feat: runtime bootstrap snapshot cache with backoff
This commit is contained in:
1 parent
08f60a14d1
commit
aa6bb23e24
19 files changed
+1400
-384
No files matched your search
+357
-32
@@ -10,18 +10,77 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pulsecast/api/internal/cache"
|
||||
"pulsecast/api/internal/config"
|
||||
"pulsecast/api/internal/ml"
|
||||
"pulsecast/api/internal/model"
|
||||
"pulsecast/api/internal/sse"
|
||||
"pulsecast/api/internal/store"
|
||||
)
|
||||
|
||||
const fixtureDir = "../../testdata/data"
|
||||
const fixtureBootstrap = "../../testdata/data/bootstrap.json"
|
||||
|
||||
type stubFetcher struct {
|
||||
mu sync.RWMutex
|
||||
payload model.Bootstrap
|
||||
err error
|
||||
refreshFn func(context.Context) (model.Bootstrap, error)
|
||||
}
|
||||
|
||||
func (s *stubFetcher) Bootstrap(ctx context.Context) (model.Bootstrap, error) {
|
||||
s.mu.RLock()
|
||||
fn := s.refreshFn
|
||||
payload, err := s.payload, s.err
|
||||
s.mu.RUnlock()
|
||||
if fn != nil {
|
||||
return fn(ctx)
|
||||
}
|
||||
if err != nil {
|
||||
return model.Bootstrap{}, err
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (s *stubFetcher) fail(err error) {
|
||||
s.mu.Lock()
|
||||
s.err = err
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *stubFetcher) recover(payload model.Bootstrap) {
|
||||
s.mu.Lock()
|
||||
s.payload, s.err = payload, nil
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func loadFixtureBootstrap(t *testing.T) model.Bootstrap {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(fixtureBootstrap)
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture: %v", err)
|
||||
}
|
||||
var payload model.Bootstrap
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
t.Fatalf("decode fixture: %v", err)
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func newFixtureCache(t *testing.T) (*cache.Cache, *stubFetcher) {
|
||||
t.Helper()
|
||||
payload := loadFixtureBootstrap(t)
|
||||
fetcher := &stubFetcher{payload: payload}
|
||||
snapshots := cache.New(fetcher, cache.Options{RefreshInterval: time.Hour, Timeout: time.Second})
|
||||
if err := snapshots.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("prime cache: %v", err)
|
||||
}
|
||||
return snapshots, fetcher
|
||||
}
|
||||
|
||||
type stubML struct {
|
||||
predict func(context.Context, ml.PredictRequest) ([]model.ForecastPoint, error)
|
||||
@@ -58,20 +117,16 @@ func (s *stubML) TrainStatus(ctx context.Context, jobID string) (int, []byte, er
|
||||
return s.trainStatus(ctx, jobID)
|
||||
}
|
||||
|
||||
func loadFixtureStore(t *testing.T) *store.Store {
|
||||
func newTestServer(t *testing.T, client MLClient, mutate func(*config.Config)) (*Server, *httptest.Server) {
|
||||
t.Helper()
|
||||
st, err := store.Load(fixtureDir)
|
||||
if err != nil {
|
||||
t.Fatalf("load fixtures: %v", err)
|
||||
}
|
||||
return st
|
||||
snapshots, _ := newFixtureCache(t)
|
||||
return newTestServerWithCache(t, snapshots, client, mutate)
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T, client MLClient, mutate func(*config.Config)) (*Server, *httptest.Server) {
|
||||
func newTestServerWithCache(t *testing.T, snapshots *cache.Cache, client MLClient, mutate func(*config.Config)) (*Server, *httptest.Server) {
|
||||
t.Helper()
|
||||
cfg := config.Config{
|
||||
Addr: ":0",
|
||||
DataDir: fixtureDir,
|
||||
FastAPIURL: "http://127.0.0.1:1",
|
||||
Heartbeat: time.Hour,
|
||||
MLTimeout: 100 * time.Millisecond,
|
||||
@@ -85,12 +140,47 @@ func newTestServer(t *testing.T, client MLClient, mutate func(*config.Config)) (
|
||||
t.Cleanup(cancel)
|
||||
go hub.Run(ctx)
|
||||
|
||||
srv := New(cfg, loadFixtureStore(t), client, hub)
|
||||
srv := New(cfg, snapshots, client, hub)
|
||||
ts := httptest.NewServer(srv.Handler())
|
||||
t.Cleanup(ts.Close)
|
||||
return srv, ts
|
||||
}
|
||||
|
||||
func forecastPoints(n int) []model.ForecastPoint {
|
||||
start := time.Date(2026, 11, 16, 9, 0, 0, 0, time.UTC)
|
||||
points := make([]model.ForecastPoint, n)
|
||||
for i := range points {
|
||||
points[i] = model.ForecastPoint{
|
||||
T: start.Add(time.Duration(i) * time.Hour),
|
||||
V: float64(10 + i),
|
||||
Lo: float64(5 + i),
|
||||
Hi: float64(15 + i),
|
||||
}
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
// weekForecast builds seven complete store-hour days (10:00-20:00 Berlin),
|
||||
// which is what the staffing planner consumes.
|
||||
func weekForecast(t *testing.T) []model.ForecastPoint {
|
||||
t.Helper()
|
||||
loc, err := time.LoadLocation(model.StoreHours.TZ)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Date(2026, 11, 16, model.StoreHours.Open, 0, 0, 0, loc)
|
||||
var points []model.ForecastPoint
|
||||
for day := 0; day < 7; day++ {
|
||||
for hour := 0; hour < model.StoreHours.Close-model.StoreHours.Open; hour++ {
|
||||
points = append(points, model.ForecastPoint{
|
||||
T: start.AddDate(0, 0, day).Add(time.Duration(hour) * time.Hour).UTC(),
|
||||
V: 25,
|
||||
})
|
||||
}
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func getJSON(t *testing.T, url string, dst any) *http.Response {
|
||||
t.Helper()
|
||||
resp, err := http.Get(url)
|
||||
@@ -125,13 +215,11 @@ func TestHealthAndReady(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadyzWithoutData(t *testing.T) {
|
||||
cfg := config.Config{Addr: ":0", DataDir: t.TempDir(), FastAPIURL: "http://127.0.0.1:1", Heartbeat: time.Hour, MLTimeout: 50 * time.Millisecond}
|
||||
st, err := store.Load(cfg.DataDir)
|
||||
if err != nil {
|
||||
t.Logf("expected load error: %v", err)
|
||||
}
|
||||
srv := New(cfg, st, &stubML{}, sse.NewHub(time.Hour))
|
||||
func TestReadyzWithoutSnapshot(t *testing.T) {
|
||||
cfg := config.Config{Addr: ":0", FastAPIURL: "http://127.0.0.1:1", Heartbeat: time.Hour, MLTimeout: 50 * time.Millisecond}
|
||||
fetcher := &stubFetcher{err: errors.New("fastapi unavailable")}
|
||||
snapshots := cache.New(fetcher, cache.Options{RefreshInterval: time.Hour, Timeout: time.Second})
|
||||
srv := New(cfg, snapshots, &stubML{}, sse.NewHub(time.Hour))
|
||||
ts := httptest.NewServer(srv.Handler())
|
||||
defer ts.Close()
|
||||
|
||||
@@ -143,6 +231,70 @@ func TestReadyzWithoutData(t *testing.T) {
|
||||
if ready["ok"] || ready["data"] || ready["model"] {
|
||||
t.Fatalf("readyz body=%v, want all false", ready)
|
||||
}
|
||||
|
||||
// One failed bootstrap attempt must not make the cache ready.
|
||||
if err := snapshots.Refresh(context.Background()); err == nil {
|
||||
t.Fatal("expected the bootstrap refresh to fail")
|
||||
}
|
||||
resp = getJSON(t, ts.URL+"/api/readyz", &ready)
|
||||
if resp.StatusCode != http.StatusServiceUnavailable || ready["ok"] {
|
||||
t.Fatalf("readyz after failed refresh: status=%d body=%v", resp.StatusCode, ready)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadyzSurvivesFastAPIFailureAfterFirstSnapshot(t *testing.T) {
|
||||
snapshots, fetcher := newFixtureCache(t)
|
||||
cfg := config.Config{Addr: ":0", FastAPIURL: "http://127.0.0.1:1", Heartbeat: time.Hour, MLTimeout: 50 * time.Millisecond}
|
||||
srv := New(cfg, snapshots, &stubML{}, sse.NewHub(time.Hour))
|
||||
ts := httptest.NewServer(srv.Handler())
|
||||
defer ts.Close()
|
||||
|
||||
var ready map[string]bool
|
||||
resp := getJSON(t, ts.URL+"/api/readyz", &ready)
|
||||
if resp.StatusCode != http.StatusOK || !ready["ok"] || !ready["data"] || !ready["model"] {
|
||||
t.Fatalf("readyz status=%d body=%v", resp.StatusCode, ready)
|
||||
}
|
||||
|
||||
fetcher.fail(errors.New("connection refused"))
|
||||
if err := snapshots.Refresh(context.Background()); err == nil {
|
||||
t.Fatal("expected the refresh to fail")
|
||||
}
|
||||
|
||||
resp = getJSON(t, ts.URL+"/api/readyz", &ready)
|
||||
if resp.StatusCode != http.StatusOK || !ready["ok"] || !ready["data"] || !ready["model"] {
|
||||
t.Fatalf("readyz after FastAPI failure: status=%d body=%v, want 200 with the last snapshot", resp.StatusCode, ready)
|
||||
}
|
||||
|
||||
var meta model.Meta
|
||||
getJSON(t, ts.URL+"/api/meta", &meta)
|
||||
if meta.Cache.Failures != 1 || meta.Cache.LastError == nil ||
|
||||
!strings.Contains(*meta.Cache.LastError, "connection refused") {
|
||||
t.Fatalf("cache status=%+v", meta.Cache)
|
||||
}
|
||||
if meta.Cache.LastSuccessAt == nil || meta.Cache.NextRetryAt == nil {
|
||||
t.Fatalf("cache timestamps missing: %+v", meta.Cache)
|
||||
}
|
||||
|
||||
// The context endpoint comes from the snapshot; it is now stale.
|
||||
var ctx model.ContextResponse
|
||||
getJSON(t, ts.URL+"/api/context", &ctx)
|
||||
if !ctx.Cached {
|
||||
t.Error("context cached=false after a failed refresh, want true")
|
||||
}
|
||||
|
||||
// A later success clears the failure state.
|
||||
fetcher.recover(loadFixtureBootstrap(t))
|
||||
if err := snapshots.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("recovery refresh: %v", err)
|
||||
}
|
||||
getJSON(t, ts.URL+"/api/meta", &meta)
|
||||
if meta.Cache.Failures != 0 || meta.Cache.LastError != nil || meta.Cache.NextRetryAt != nil {
|
||||
t.Fatalf("cache status after recovery=%+v", meta.Cache)
|
||||
}
|
||||
getJSON(t, ts.URL+"/api/context", &ctx)
|
||||
if ctx.Cached {
|
||||
t.Error("context cached=true after a successful refresh, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetaSensorsAndVersionHeaders(t *testing.T) {
|
||||
@@ -156,6 +308,10 @@ func TestMetaSensorsAndVersionHeaders(t *testing.T) {
|
||||
if meta.Coverage.Sensors != 8 || meta.ModelVersion == "" || meta.ContextVersion == "" {
|
||||
t.Fatalf("meta=%+v", meta)
|
||||
}
|
||||
if meta.Cache.Source != "fastapi" || meta.Cache.LastSuccessAt == nil ||
|
||||
meta.Cache.LastError != nil || meta.Cache.NextRetryAt != nil || meta.Cache.Failures != 0 {
|
||||
t.Fatalf("meta cache=%+v", meta.Cache)
|
||||
}
|
||||
for _, header := range []string{"X-Data-Version", "X-Model-Version", "X-Context-Version"} {
|
||||
if resp.Header.Get(header) == "" {
|
||||
t.Errorf("missing %s header", header)
|
||||
@@ -180,6 +336,58 @@ func TestMetaSensorsAndVersionHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataEndpointsFollowSnapshotRefresh(t *testing.T) {
|
||||
snapshots, fetcher := newFixtureCache(t)
|
||||
_, ts := newTestServerWithCache(t, snapshots, &stubML{}, nil)
|
||||
|
||||
var sensors []model.Sensor
|
||||
getJSON(t, ts.URL+"/api/sensors", &sensors)
|
||||
if len(sensors) != 8 {
|
||||
t.Fatalf("sensors=%d, want 8", len(sensors))
|
||||
}
|
||||
|
||||
updated := loadFixtureBootstrap(t)
|
||||
updated.DataVersion = "2026-11-14T05:00:00Z"
|
||||
updated.ContextVersion = "2026-11-14T05:00:00Z"
|
||||
updated.Sensors = updated.Sensors[:1]
|
||||
updated.Series = map[string][]model.Point{
|
||||
updated.Sensors[0].ID: {{T: time.Date(2026, 11, 1, 10, 0, 0, 0, time.UTC), V: 42}},
|
||||
}
|
||||
fetcher.recover(updated)
|
||||
if err := snapshots.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("refresh: %v", err)
|
||||
}
|
||||
|
||||
sensors = nil
|
||||
resp := getJSON(t, ts.URL+"/api/sensors", &sensors)
|
||||
if len(sensors) != 1 || sensors[0].ID != updated.Sensors[0].ID {
|
||||
t.Fatalf("sensors after refresh=%+v", sensors)
|
||||
}
|
||||
if got := resp.Header.Get("X-Data-Version"); got != updated.DataVersion {
|
||||
t.Errorf("X-Data-Version=%q, want %q", got, updated.DataVersion)
|
||||
}
|
||||
|
||||
var series struct {
|
||||
Points []model.Point `json:"points"`
|
||||
}
|
||||
getJSON(t, ts.URL+"/api/series?sensor="+updated.Sensors[0].ID+"&step=15m", &series)
|
||||
if len(series.Points) != 1 || series.Points[0].V != 42 {
|
||||
t.Fatalf("series after refresh=%+v", series.Points)
|
||||
}
|
||||
|
||||
var ctx model.ContextResponse
|
||||
getJSON(t, ts.URL+"/api/context", &ctx)
|
||||
if ctx.UpdatedAt.IsZero() || ctx.Cached {
|
||||
t.Fatalf("context after refresh=%+v", ctx)
|
||||
}
|
||||
|
||||
var meta model.Meta
|
||||
getJSON(t, ts.URL+"/api/meta", &meta)
|
||||
if meta.DataVersion != updated.DataVersion || meta.Cache.Failures != 0 {
|
||||
t.Fatalf("meta after refresh=%+v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesRangeStepAndErrors(t *testing.T) {
|
||||
_, ts := newTestServer(t, &stubML{}, nil)
|
||||
|
||||
@@ -241,37 +449,96 @@ func TestContextEndpoint(t *testing.T) {
|
||||
if ctx.UpdatedAt.IsZero() {
|
||||
t.Error("expected updated_at")
|
||||
}
|
||||
if ctx.Cached {
|
||||
t.Error("cached=true while the last snapshot refresh succeeded, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestForecastFallsBackWhenMLIsDown(t *testing.T) {
|
||||
_, ts := newTestServer(t, &stubML{}, nil)
|
||||
func TestForecastServesLastGoodWhenMLIsDown(t *testing.T) {
|
||||
live := forecastPoints(168)
|
||||
var down bool
|
||||
var mu sync.Mutex
|
||||
client := &stubML{predict: func(context.Context, ml.PredictRequest) ([]model.ForecastPoint, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if down {
|
||||
return nil, errors.New("connection refused")
|
||||
}
|
||||
return live, nil
|
||||
}}
|
||||
_, ts := newTestServer(t, client, nil)
|
||||
|
||||
var forecast model.ForecastResponse
|
||||
resp := getJSON(t, ts.URL+"/api/forecast?sensor=hn-bahnhof-01&horizon=7d", &forecast)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("forecast status=%d", resp.StatusCode)
|
||||
t.Fatalf("live forecast status=%d", resp.StatusCode)
|
||||
}
|
||||
if !forecast.Cached {
|
||||
if forecast.Cached || len(forecast.Points) != 168 {
|
||||
t.Fatalf("live forecast cached=%v points=%d, want false/168", forecast.Cached, len(forecast.Points))
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
down = true
|
||||
mu.Unlock()
|
||||
|
||||
var fallback model.ForecastResponse
|
||||
resp = getJSON(t, ts.URL+"/api/forecast?sensor=hn-bahnhof-01&horizon=7d", &fallback)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("fallback status=%d", resp.StatusCode)
|
||||
}
|
||||
if !fallback.Cached {
|
||||
t.Error("cached=false, want true when FastAPI is down")
|
||||
}
|
||||
if len(forecast.Points) != 168 {
|
||||
t.Fatalf("points=%d, want 168", len(forecast.Points))
|
||||
if len(fallback.Points) != 168 || fallback.Points[0].T != forecast.Points[0].T ||
|
||||
fallback.Points[0].V != forecast.Points[0].V {
|
||||
t.Fatalf("fallback points do not match the last good forecast: %+v", fallback.Points[0])
|
||||
}
|
||||
if forecast.Scenario != "base" || forecast.DataVersion == "" || forecast.ModelVersion == "" {
|
||||
t.Fatalf("forecast=%+v", forecast)
|
||||
if fallback.Scenario != "base" || fallback.DataVersion == "" || fallback.ModelVersion == "" {
|
||||
t.Fatalf("forecast=%+v", fallback)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForecastTimeoutFallsBack(t *testing.T) {
|
||||
func TestForecastWithoutLastGoodFails503(t *testing.T) {
|
||||
_, ts := newTestServer(t, &stubML{}, nil)
|
||||
var errBody model.ErrorBody
|
||||
resp := getJSON(t, ts.URL+"/api/forecast?sensor=hn-bahnhof-01&horizon=7d", &errBody)
|
||||
if resp.StatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d, want 503", resp.StatusCode)
|
||||
}
|
||||
if errBody.Error.Code != "forecast_unavailable" {
|
||||
t.Fatalf("error=%+v", errBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForecastTimeoutServesLastGood(t *testing.T) {
|
||||
live := forecastPoints(48)
|
||||
var mu sync.Mutex
|
||||
slow := false
|
||||
client := &stubML{predict: func(ctx context.Context, _ ml.PredictRequest) ([]model.ForecastPoint, error) {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
mu.Lock()
|
||||
block := slow
|
||||
mu.Unlock()
|
||||
if block {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return live, nil
|
||||
}}
|
||||
_, ts := newTestServer(t, client, func(cfg *config.Config) { cfg.MLTimeout = 20 * time.Millisecond })
|
||||
|
||||
var primed model.ForecastResponse
|
||||
resp := getJSON(t, ts.URL+"/api/forecast?sensor=hn-stadtgalerie-01&horizon=48h&scenario=sunny-saturday-market", &primed)
|
||||
if resp.StatusCode != http.StatusOK || primed.Cached {
|
||||
t.Fatalf("prime forecast status=%d cached=%v", resp.StatusCode, primed.Cached)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
slow = true
|
||||
mu.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
var forecast model.ForecastResponse
|
||||
resp := getJSON(t, ts.URL+"/api/forecast?sensor=hn-stadtgalerie-01&horizon=48h&scenario=sunny-saturday-market", &forecast)
|
||||
resp = getJSON(t, ts.URL+"/api/forecast?sensor=hn-stadtgalerie-01&horizon=48h&scenario=sunny-saturday-market", &forecast)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("forecast status=%d", resp.StatusCode)
|
||||
}
|
||||
@@ -327,7 +594,11 @@ func TestForecastValidation(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStaffingPlan(t *testing.T) {
|
||||
_, ts := newTestServer(t, &stubML{}, nil)
|
||||
points := weekForecast(t)
|
||||
client := &stubML{predict: func(context.Context, ml.PredictRequest) ([]model.ForecastPoint, error) {
|
||||
return points, nil
|
||||
}}
|
||||
_, ts := newTestServer(t, client, nil)
|
||||
|
||||
start := time.Now()
|
||||
var plan model.StaffingResponse
|
||||
@@ -350,9 +621,12 @@ func TestStaffingPlan(t *testing.T) {
|
||||
if plan.Summary.CoveragePct != 100 {
|
||||
t.Errorf("coverage=%v, want 100", plan.Summary.CoveragePct)
|
||||
}
|
||||
if plan.Summary.CostEUR <= 0 || plan.Summary.FlatScheduleCostEUR <= 0 {
|
||||
if plan.Summary.CostEUR != 2240 || plan.Summary.FlatScheduleCostEUR != 2240 {
|
||||
t.Errorf("summary=%+v", plan.Summary)
|
||||
}
|
||||
if plan.Cached {
|
||||
t.Error("cached=true, want false for a live plan")
|
||||
}
|
||||
if len(plan.Why) == 0 {
|
||||
t.Error("expected a non-empty why list")
|
||||
}
|
||||
@@ -363,6 +637,57 @@ func TestStaffingPlan(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaffingFallsBackToLastGoodForecast(t *testing.T) {
|
||||
points := weekForecast(t)
|
||||
var mu sync.Mutex
|
||||
down := false
|
||||
client := &stubML{predict: func(context.Context, ml.PredictRequest) ([]model.ForecastPoint, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if down {
|
||||
return nil, errors.New("connection refused")
|
||||
}
|
||||
return points, nil
|
||||
}}
|
||||
_, ts := newTestServer(t, client, nil)
|
||||
|
||||
var live model.StaffingResponse
|
||||
resp := getJSON(t, ts.URL+"/api/staffing?sensor=hn-bahnhof-01&scenario=base&standard=standard", &live)
|
||||
if resp.StatusCode != http.StatusOK || live.Cached {
|
||||
t.Fatalf("live staffing status=%d cached=%v", resp.StatusCode, live.Cached)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
down = true
|
||||
mu.Unlock()
|
||||
|
||||
// Different standard -> different plan key, so the handler must rebuild
|
||||
// from the last good 7d forecast.
|
||||
var fallback model.StaffingResponse
|
||||
resp = getJSON(t, ts.URL+"/api/staffing?sensor=hn-bahnhof-01&scenario=base&standard=premium", &fallback)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("fallback staffing status=%d", resp.StatusCode)
|
||||
}
|
||||
if !fallback.Cached {
|
||||
t.Error("cached=false, want true when built from the last good forecast")
|
||||
}
|
||||
if len(fallback.Hours) != 70 || len(fallback.Shifts) != 21 {
|
||||
t.Fatalf("fallback plan hours=%d shifts=%d", len(fallback.Hours), len(fallback.Shifts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaffingUnavailableWithoutForecast(t *testing.T) {
|
||||
_, ts := newTestServer(t, &stubML{}, nil)
|
||||
var errBody model.ErrorBody
|
||||
resp := getJSON(t, ts.URL+"/api/staffing?sensor=hn-bahnhof-01&scenario=base&standard=standard", &errBody)
|
||||
if resp.StatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d, want 503", resp.StatusCode)
|
||||
}
|
||||
if errBody.Error.Code != "staffing_unavailable" {
|
||||
t.Fatalf("error=%+v", errBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaffingValidation(t *testing.T) {
|
||||
_, ts := newTestServer(t, &stubML{}, nil)
|
||||
for _, tc := range []struct {
|
||||
|
||||
@@ -14,9 +14,11 @@ func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleReadyz(w http.ResponseWriter, _ *http.Request) {
|
||||
ready := s.store != nil && s.store.Ready()
|
||||
dataOK := s.store != nil && s.store.Len() > 0 && !s.store.Context().UpdatedAt.IsZero()
|
||||
modelOK := s.store != nil && s.store.Meta().ModelVersion != ""
|
||||
dataOK, modelOK := false, false
|
||||
if s.cache != nil {
|
||||
dataOK, modelOK = s.cache.Readiness()
|
||||
}
|
||||
ready := dataOK && modelOK
|
||||
status := http.StatusOK
|
||||
if !ready {
|
||||
status = http.StatusServiceUnavailable
|
||||
@@ -26,12 +28,16 @@ func (s *Server) handleReadyz(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
func (s *Server) handleMeta(w http.ResponseWriter, _ *http.Request) {
|
||||
s.setVersions(w)
|
||||
writeJSON(w, http.StatusOK, s.store.Meta())
|
||||
writeJSON(w, http.StatusOK, s.meta())
|
||||
}
|
||||
|
||||
func (s *Server) handleSensors(w http.ResponseWriter, _ *http.Request) {
|
||||
s.setVersions(w)
|
||||
writeJSON(w, http.StatusOK, s.store.Sensors())
|
||||
if s.cache == nil {
|
||||
writeJSON(w, http.StatusOK, []model.Sensor{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, s.cache.Sensors())
|
||||
}
|
||||
|
||||
func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -41,7 +47,11 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
|
||||
return
|
||||
}
|
||||
if _, ok := s.store.Sensor(sensor); !ok {
|
||||
if s.cache == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
if _, ok := s.cache.Sensor(sensor); !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
@@ -54,7 +64,7 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
raw, _ := s.store.Series(sensor)
|
||||
raw, _ := s.cache.Series(sensor)
|
||||
if len(raw) == 0 {
|
||||
s.setVersions(w)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"sensor": sensor, "step": step, "points": []model.Point{}})
|
||||
@@ -69,7 +79,7 @@ func (s *Server) handleSeries(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
to = parsed.UTC()
|
||||
} else if len(raw) > 0 {
|
||||
} else {
|
||||
to = raw[len(raw)-1].T.UTC()
|
||||
}
|
||||
|
||||
@@ -143,12 +153,17 @@ func hourlyMean(points []model.Point) []model.Point {
|
||||
}
|
||||
|
||||
func (s *Server) handleContext(w http.ResponseWriter, _ *http.Request) {
|
||||
ctx := s.store.Context()
|
||||
var ctx model.ContextData
|
||||
cached := true
|
||||
if s.cache != nil {
|
||||
ctx = s.cache.Context()
|
||||
cached = !s.cache.Fresh()
|
||||
}
|
||||
s.setVersions(w)
|
||||
writeJSON(w, http.StatusOK, model.ContextResponse{
|
||||
Weather: ctx.Weather,
|
||||
Holidays: ctx.Holidays,
|
||||
UpdatedAt: ctx.UpdatedAt,
|
||||
Cached: false,
|
||||
Cached: cached,
|
||||
})
|
||||
}
|
||||
@@ -20,7 +20,11 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
|
||||
return
|
||||
}
|
||||
if _, ok := s.store.Sensor(sensor); !ok {
|
||||
if s.cache == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
if _, ok := s.cache.Sensor(sensor); !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
@@ -53,11 +57,14 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
slog.Warn("forecast unavailable", "sensor", sensor, "horizon", horizon, "scenario", scenarioName, "error", err)
|
||||
writeError(w, http.StatusServiceUnavailable, "forecast_unavailable",
|
||||
"no live forecast and no baked forecast for this sensor, scenario and horizon")
|
||||
"no live forecast and no last good forecast for this sensor, scenario and horizon")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
s.rememberForecast(sensor, horizon, scenarioName, points)
|
||||
}
|
||||
|
||||
meta := s.meta()
|
||||
s.setVersions(w)
|
||||
writeJSON(w, http.StatusOK, model.ForecastResponse{
|
||||
Sensor: sensor,
|
||||
@@ -65,9 +72,9 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
|
||||
Scenario: scenarioName,
|
||||
Points: points,
|
||||
Cached: cached,
|
||||
DataVersion: s.store.Meta().DataVersion,
|
||||
ModelVersion: s.store.Meta().ModelVersion,
|
||||
ContextVersion: s.store.Meta().ContextVersion,
|
||||
DataVersion: meta.DataVersion,
|
||||
ModelVersion: meta.ModelVersion,
|
||||
ContextVersion: meta.ContextVersion,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -78,7 +85,7 @@ func (s *Server) liveForecast(ctx context.Context, sensor, horizon, scenarioName
|
||||
ctx, cancel := context.WithTimeout(ctx, s.cfg.MLTimeout)
|
||||
defer cancel()
|
||||
|
||||
ctxData := s.store.Context()
|
||||
ctxData := s.cache.Context()
|
||||
precip := ctxData.Weather.Now.PrecipMM
|
||||
if state.Scenario.Weather.PrecipMM > 0 {
|
||||
precip = state.Scenario.Weather.PrecipMM
|
||||
@@ -100,8 +107,9 @@ func (s *Server) liveForecast(ctx context.Context, sensor, horizon, scenarioName
|
||||
return sortedForecast(points), nil
|
||||
}
|
||||
|
||||
// fallbackForecast prefers live /simulate output for the current scenario, then
|
||||
// the baked scenario grid. It never labels base data as another scenario.
|
||||
// fallbackForecast prefers the live /simulate grid for the current scenario,
|
||||
// then the last good /predict response for this exact key. It never labels base
|
||||
// data as another scenario.
|
||||
func (s *Server) fallbackForecast(sensor, scenarioName, horizon string) ([]model.ForecastPoint, bool) {
|
||||
s.mu.RLock()
|
||||
if s.simFor == scenarioName {
|
||||
@@ -115,10 +123,31 @@ func (s *Server) fallbackForecast(sensor, scenarioName, horizon string) ([]model
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
if points, ok := s.store.Forecast(sensor, scenarioName, horizon); ok {
|
||||
return sortedForecast(points), true
|
||||
return s.lastGoodForecast(sensor, horizon, scenarioName)
|
||||
}
|
||||
|
||||
func forecastKey(sensor, horizon, scenarioName string) string {
|
||||
return sensor + "|" + horizon + "|" + scenarioName
|
||||
}
|
||||
|
||||
func (s *Server) rememberForecast(sensor, horizon, scenarioName string, points []model.ForecastPoint) {
|
||||
if len(points) == 0 {
|
||||
return
|
||||
}
|
||||
return nil, false
|
||||
copied := sortedForecast(points)
|
||||
s.mu.Lock()
|
||||
s.lastForecast[forecastKey(sensor, horizon, scenarioName)] = copied
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *Server) lastGoodForecast(sensor, horizon, scenarioName string) ([]model.ForecastPoint, bool) {
|
||||
s.mu.RLock()
|
||||
points, ok := s.lastForecast[forecastKey(sensor, horizon, scenarioName)]
|
||||
s.mu.RUnlock()
|
||||
if !ok || len(points) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
return sortedForecast(points), true
|
||||
}
|
||||
|
||||
func sortedForecast(points []model.ForecastPoint) []model.ForecastPoint {
|
||||
@@ -129,8 +158,11 @@ func sortedForecast(points []model.ForecastPoint) []model.ForecastPoint {
|
||||
}
|
||||
|
||||
func (s *Server) holidayOn(t time.Time) (string, bool) {
|
||||
if s.cache == nil {
|
||||
return "", false
|
||||
}
|
||||
day := t.In(s.loc).Format("2006-01-02")
|
||||
for _, holiday := range s.store.Context().Holidays {
|
||||
for _, holiday := range s.cache.Context().Holidays {
|
||||
if holiday.Date == day {
|
||||
return holiday.Name, true
|
||||
}
|
||||
@@ -145,7 +177,11 @@ func (s *Server) handleStaffing(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
|
||||
return
|
||||
}
|
||||
if _, ok := s.store.Sensor(sensor); !ok {
|
||||
if s.cache == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
if _, ok := s.cache.Sensor(sensor); !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
@@ -169,45 +205,49 @@ func (s *Server) handleStaffing(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
state := s.scenario.Get()
|
||||
points, cached, ok := s.staffingForecast(sensor, scenarioName)
|
||||
if !ok {
|
||||
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable",
|
||||
"no 7d forecast for this sensor and scenario")
|
||||
return
|
||||
}
|
||||
meta := s.meta()
|
||||
|
||||
s.mu.RLock()
|
||||
generation := s.simGen
|
||||
s.mu.RUnlock()
|
||||
key := fmt.Sprintf("%s|%s|%s|%d|%d", sensor, scenarioName, standard, state.Version, generation)
|
||||
key := fmt.Sprintf("%s|%s|%s|%s|%d|%d", sensor, scenarioName, standard,
|
||||
meta.ContextVersion, state.Version, generation)
|
||||
|
||||
s.mu.RLock()
|
||||
plan, hit := s.staffing[key]
|
||||
s.mu.RUnlock()
|
||||
|
||||
if !hit {
|
||||
built, err := staffing.Build(sensor, scenarioName, standard, points, s.store.Context())
|
||||
points, cached, ok := s.staffingForecast(r.Context(), sensor, scenarioName, state)
|
||||
if !ok {
|
||||
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable",
|
||||
"no live or last good 7d forecast for this sensor and scenario")
|
||||
return
|
||||
}
|
||||
built, err := staffing.Build(sensor, scenarioName, standard, points, s.cache.Context())
|
||||
if err != nil {
|
||||
slog.Warn("staffing build failed", "sensor", sensor, "scenario", scenarioName, "standard", standard, "error", err)
|
||||
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable", "could not build a plan from the available forecast")
|
||||
return
|
||||
}
|
||||
built.Cached = cached
|
||||
s.mu.Lock()
|
||||
s.staffing[key] = built
|
||||
s.mu.Unlock()
|
||||
plan = built
|
||||
}
|
||||
|
||||
plan.Cached = cached
|
||||
plan.DataVersion = s.store.Meta().DataVersion
|
||||
plan.ModelVersion = s.store.Meta().ModelVersion
|
||||
plan.ContextVersion = s.store.Meta().ContextVersion
|
||||
plan.DataVersion = meta.DataVersion
|
||||
plan.ModelVersion = meta.ModelVersion
|
||||
plan.ContextVersion = meta.ContextVersion
|
||||
s.setVersions(w)
|
||||
writeJSON(w, http.StatusOK, plan)
|
||||
}
|
||||
|
||||
// staffingForecast returns the best 7d points and whether they came from the
|
||||
// fallback grid rather than live /simulate output.
|
||||
func (s *Server) staffingForecast(sensor, scenarioName string) ([]model.ForecastPoint, bool, bool) {
|
||||
// staffingForecast returns 7d points for a plan. Prefer the scenario simulation
|
||||
// grid, then /predict, and finally the last good forecast. The bool reports
|
||||
// whether the points came from the cache fallback.
|
||||
func (s *Server) staffingForecast(ctx context.Context, sensor, scenarioName string, state model.ScenarioState) ([]model.ForecastPoint, bool, bool) {
|
||||
s.mu.RLock()
|
||||
if s.simFor == scenarioName {
|
||||
if horizons, ok := s.sim[sensor]; ok {
|
||||
@@ -220,8 +260,14 @@ func (s *Server) staffingForecast(sensor, scenarioName string) ([]model.Forecast
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
if points, ok := s.store.Forecast(sensor, scenarioName, "7d"); ok {
|
||||
return sortedForecast(points), scenarioName != "base", true
|
||||
points, err := s.liveForecast(ctx, sensor, "7d", scenarioName, state)
|
||||
if err == nil {
|
||||
s.rememberForecast(sensor, "7d", scenarioName, points)
|
||||
return points, false, true
|
||||
}
|
||||
slog.Warn("staffing forecast fallback", "sensor", sensor, "scenario", scenarioName, "error", err)
|
||||
if points, ok := s.lastGoodForecast(sensor, "7d", scenarioName); ok {
|
||||
return points, true, true
|
||||
}
|
||||
return nil, false, false
|
||||
}
|
||||
@@ -83,6 +83,14 @@ func (s *Server) refreshSimulation(state model.ScenarioState) {
|
||||
s.simFor = state.Scenario.Scenario
|
||||
s.simGen++
|
||||
s.staffing = make(map[string]model.StaffingResponse)
|
||||
for sensorID, horizons := range forecasts {
|
||||
for horizon, points := range horizons {
|
||||
if len(points) == 0 {
|
||||
continue
|
||||
}
|
||||
s.lastForecast[forecastKey(sensorID, horizon, state.Scenario.Scenario)] = sortedForecast(points)
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
|
||||
+31
-20
@@ -12,12 +12,12 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pulsecast/api/internal/cache"
|
||||
"pulsecast/api/internal/config"
|
||||
"pulsecast/api/internal/ml"
|
||||
"pulsecast/api/internal/model"
|
||||
"pulsecast/api/internal/scenario"
|
||||
"pulsecast/api/internal/sse"
|
||||
"pulsecast/api/internal/store"
|
||||
)
|
||||
|
||||
// MLClient is the FastAPI prediction surface the data handlers need.
|
||||
@@ -37,22 +37,23 @@ type TrainClient interface {
|
||||
// Server holds process-wide state. One instance per pod.
|
||||
type Server struct {
|
||||
cfg config.Config
|
||||
store *store.Store
|
||||
cache *cache.Cache
|
||||
ml MLClient
|
||||
hub *sse.Hub
|
||||
scenario *scenario.State
|
||||
loc *time.Location
|
||||
now func() time.Time
|
||||
|
||||
mu sync.RWMutex
|
||||
sim map[string]map[string][]model.ForecastPoint // sensor -> horizon -> points
|
||||
simFor string // scenario the sim belongs to
|
||||
simGen int // invalidates staffing cache
|
||||
staffing map[string]model.StaffingResponse
|
||||
mu sync.RWMutex
|
||||
sim map[string]map[string][]model.ForecastPoint // sensor -> horizon -> points
|
||||
simFor string // scenario the sim belongs to
|
||||
simGen int // invalidates staffing cache
|
||||
staffing map[string]model.StaffingResponse
|
||||
lastForecast map[string][]model.ForecastPoint // sensor|horizon|scenario -> last live points
|
||||
}
|
||||
|
||||
// New builds a server. hub must be non-nil; main runs hub.Run.
|
||||
func New(cfg config.Config, st *store.Store, client MLClient, hub *sse.Hub) *Server {
|
||||
func New(cfg config.Config, snapshots *cache.Cache, client MLClient, hub *sse.Hub) *Server {
|
||||
loc, err := time.LoadLocation(model.StoreHours.TZ)
|
||||
if err != nil {
|
||||
loc = time.UTC
|
||||
@@ -61,14 +62,15 @@ func New(cfg config.Config, st *store.Store, client MLClient, hub *sse.Hub) *Ser
|
||||
cfg.MLTimeout = ml.DefaultTimeout
|
||||
}
|
||||
return &Server{
|
||||
cfg: cfg,
|
||||
store: st,
|
||||
ml: client,
|
||||
hub: hub,
|
||||
scenario: scenario.New(),
|
||||
loc: loc,
|
||||
now: time.Now,
|
||||
staffing: make(map[string]model.StaffingResponse),
|
||||
cfg: cfg,
|
||||
cache: snapshots,
|
||||
ml: client,
|
||||
hub: hub,
|
||||
scenario: scenario.New(),
|
||||
loc: loc,
|
||||
now: time.Now,
|
||||
staffing: make(map[string]model.StaffingResponse),
|
||||
lastForecast: make(map[string][]model.ForecastPoint),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,10 +196,19 @@ func writeError(w http.ResponseWriter, status int, code, message string) {
|
||||
_, _ = w.Write(raw)
|
||||
}
|
||||
|
||||
// meta snapshots the cache metadata once per response, so versions and the
|
||||
// cache block cannot disagree when a refresh lands mid-request.
|
||||
func (s *Server) meta() model.Meta {
|
||||
if s.cache == nil {
|
||||
return model.Meta{}
|
||||
}
|
||||
return s.cache.Meta()
|
||||
}
|
||||
|
||||
// setVersions adds the contract's version headers.
|
||||
func (s *Server) setVersions(w http.ResponseWriter) {
|
||||
data, modelVersion, context := s.store.Versions()
|
||||
w.Header().Set("X-Data-Version", data)
|
||||
w.Header().Set("X-Model-Version", modelVersion)
|
||||
w.Header().Set("X-Context-Version", context)
|
||||
meta := s.meta()
|
||||
w.Header().Set("X-Data-Version", meta.DataVersion)
|
||||
w.Header().Set("X-Model-Version", meta.ModelVersion)
|
||||
w.Header().Set("X-Context-Version", meta.ContextVersion)
|
||||
}
|
||||
Reference in new issue
Block a user