feat: runtime bootstrap snapshot cache with backoff

This commit is contained in:
Noel Sabia committed 2026-11-14 06:08:53 +01:00
1 parent 08f60a14d1
commit aa6bb23e24
19 files changed
+1400 -384

No files matched your search

+358
View File
@@ -0,0 +1,358 @@
// Package cache keeps the FastAPI /bootstrap snapshot in memory and refreshes
// it in the background. A failed refresh never discards the last good snapshot;
// the next attempt follows the contract backoff 1/2/4/8/16/30 s (cap).
package cache
import (
"context"
"fmt"
"log/slog"
"sort"
"sync"
"time"
"pulsecast/api/internal/model"
)
// Source is the only snapshot producer: the FastAPI /bootstrap endpoint.
const Source = "fastapi"
// DefaultRefreshInterval is how often a healthy snapshot is refreshed.
const DefaultRefreshInterval = 60 * time.Second
// DefaultTimeout bounds one /bootstrap request.
const DefaultTimeout = 20 * time.Second
// DefaultRetryDelays is the contract backoff schedule. The last value repeats.
var DefaultRetryDelays = []time.Duration{
1 * time.Second,
2 * time.Second,
4 * time.Second,
8 * time.Second,
16 * time.Second,
30 * time.Second,
}
// Fetcher is the FastAPI bootstrap surface. *ml.Client implements it.
type Fetcher interface {
Bootstrap(ctx context.Context) (model.Bootstrap, error)
}
// Options configures refresh behaviour. Zero values use the defaults.
type Options struct {
RefreshInterval time.Duration
Timeout time.Duration
}
// snapshot is one immutable /bootstrap payload with its lookup indexes.
type snapshot struct {
DataVersion string
ModelVersion string
ContextVersion string
Sensors []model.Sensor
Series map[string][]model.Point
Context model.ContextData
Coverage model.Coverage
Metrics model.Metrics
Limits []string
sensorBy map[string]model.Sensor
}
// Cache holds the current snapshot and its refresh status. All methods are
// safe for concurrent use.
type Cache struct {
fetcher Fetcher
refresh time.Duration
timeout time.Duration
retry []time.Duration
now func() time.Time
mu sync.RWMutex
snapshot *snapshot
loaded bool
status model.CacheStatus
}
// New returns an empty cache. Call Run (or Refresh) to load the first snapshot.
func New(fetcher Fetcher, opts Options) *Cache {
if opts.RefreshInterval <= 0 {
opts.RefreshInterval = DefaultRefreshInterval
}
if opts.Timeout <= 0 {
opts.Timeout = DefaultTimeout
}
return &Cache{
fetcher: fetcher,
refresh: opts.RefreshInterval,
timeout: opts.Timeout,
retry: append([]time.Duration(nil), DefaultRetryDelays...),
now: time.Now,
status: model.CacheStatus{Source: Source},
}
}
// Run loads the snapshot immediately, then refreshes it. Failures keep the last
// good snapshot and retry with the contract backoff. Run returns when ctx ends.
func (c *Cache) Run(ctx context.Context) {
recovering := false
for {
wasLoaded := c.Loaded()
err := c.Refresh(ctx)
if ctx.Err() != nil {
return
}
var delay time.Duration
if err != nil {
recovering = true
status := c.Status()
delay = c.retryDelay(status.Failures)
slog.Warn("bootstrap refresh failed; keeping last good snapshot",
"error", err, "failures", status.Failures, "retry_in", delay)
} else {
delay = c.refresh
switch {
case !wasLoaded:
slog.Info("bootstrap snapshot loaded", "sensors", c.Len())
case recovering:
slog.Info("bootstrap refresh recovered", "sensors", c.Len())
}
recovering = false
}
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
}
}
// Refresh performs one fetch. A decode/validation failure counts as a failed
// attempt and leaves the previous snapshot untouched.
func (c *Cache) Refresh(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()
payload, err := c.fetcher.Bootstrap(ctx)
now := c.now().UTC().Truncate(time.Second)
if err == nil {
var snapshot *snapshot
snapshot, err = newSnapshot(payload)
if err == nil {
c.mu.Lock()
c.snapshot = snapshot
c.loaded = true
c.status.LastSuccessAt = timePtr(now)
c.status.LastError = nil
c.status.NextRetryAt = nil
c.status.Failures = 0
c.mu.Unlock()
return nil
}
}
c.mu.Lock()
c.status.Failures++
message := err.Error()
c.status.LastError = &message
next := now.Add(c.retryDelay(c.status.Failures))
c.status.NextRetryAt = &next
c.mu.Unlock()
return err
}
func (c *Cache) retryDelay(failures int) time.Duration {
if failures < 1 {
failures = 1
}
if failures > len(c.retry) {
failures = len(c.retry)
}
return c.retry[failures-1]
}
func newSnapshot(payload model.Bootstrap) (*snapshot, error) {
switch {
case len(payload.Sensors) == 0:
return nil, fmt.Errorf("bootstrap snapshot: no sensors")
case payload.DataVersion == "":
return nil, fmt.Errorf("bootstrap snapshot: missing data_version")
case payload.ModelVersion == "":
return nil, fmt.Errorf("bootstrap snapshot: missing model_version")
case payload.ContextVersion == "":
return nil, fmt.Errorf("bootstrap snapshot: missing context_version")
case payload.Context.UpdatedAt.IsZero():
return nil, fmt.Errorf("bootstrap snapshot: missing context.updated_at")
}
snapshot := &snapshot{
DataVersion: payload.DataVersion,
ModelVersion: payload.ModelVersion,
ContextVersion: payload.ContextVersion,
Sensors: append([]model.Sensor(nil), payload.Sensors...),
Series: make(map[string][]model.Point, len(payload.Series)),
Context: payload.Context,
Coverage: payload.Meta.Coverage,
Metrics: payload.Meta.Metrics,
Limits: append([]string(nil), payload.Meta.Limits...),
sensorBy: make(map[string]model.Sensor, len(payload.Sensors)),
}
sort.Slice(snapshot.Sensors, func(i, j int) bool { return snapshot.Sensors[i].ID < snapshot.Sensors[j].ID })
for _, sensor := range snapshot.Sensors {
snapshot.sensorBy[sensor.ID] = sensor
}
for sensorID, points := range payload.Series {
copied := append([]model.Point(nil), points...)
sort.Slice(copied, func(i, j int) bool { return copied[i].T.Before(copied[j].T) })
snapshot.Series[sensorID] = copied
}
snapshot.Context.Holidays = append([]model.Holiday(nil), payload.Context.Holidays...)
snapshot.Context.Weather.Forecast = append([]model.WeatherPoint(nil), payload.Context.Weather.Forecast...)
return snapshot, nil
}
// Loaded reports whether a snapshot has ever been accepted.
func (c *Cache) Loaded() bool {
c.mu.RLock()
defer c.mu.RUnlock()
return c.loaded
}
// Ready reports whether a complete snapshot is available. FastAPI health is
// deliberately not part of this check: the cache itself is the safety net.
func (c *Cache) Ready() bool {
data, modelOK := c.Readiness()
return data && modelOK
}
// Readiness splits readiness into the data and model signals of /api/readyz.
func (c *Cache) Readiness() (data, modelOK bool) {
c.mu.RLock()
defer c.mu.RUnlock()
if !c.loaded || c.snapshot == nil {
return false, false
}
data = len(c.snapshot.Sensors) > 0 && !c.snapshot.Context.UpdatedAt.IsZero()
modelOK = c.snapshot.ModelVersion != ""
return data, modelOK
}
// Fresh reports whether the most recent refresh attempt succeeded. It is false
// before the first snapshot and after a failed refresh.
func (c *Cache) Fresh() bool {
c.mu.RLock()
defer c.mu.RUnlock()
return c.loaded && c.status.Failures == 0
}
// Status returns a copy of the cache status.
func (c *Cache) Status() model.CacheStatus {
c.mu.RLock()
defer c.mu.RUnlock()
return cloneStatus(c.status)
}
// Meta combines the snapshot metadata with the live cache status.
func (c *Cache) Meta() model.Meta {
c.mu.RLock()
defer c.mu.RUnlock()
meta := model.Meta{Cache: cloneStatus(c.status)}
if c.snapshot == nil {
return meta
}
meta.DataVersion = c.snapshot.DataVersion
meta.ModelVersion = c.snapshot.ModelVersion
meta.ContextVersion = c.snapshot.ContextVersion
meta.Coverage = c.snapshot.Coverage
meta.Metrics = c.snapshot.Metrics
meta.Limits = append([]string(nil), c.snapshot.Limits...)
return meta
}
// Sensors returns the sensors sorted by id.
func (c *Cache) Sensors() []model.Sensor {
c.mu.RLock()
snapshot := c.snapshot
c.mu.RUnlock()
if snapshot == nil {
return []model.Sensor{}
}
out := make([]model.Sensor, len(snapshot.Sensors))
copy(out, snapshot.Sensors)
return out
}
// Sensor looks up one sensor.
func (c *Cache) Sensor(id string) (model.Sensor, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return model.Sensor{}, false
}
sensor, ok := c.snapshot.sensorBy[id]
return sensor, ok
}
// Series returns the immutable series for a sensor, sorted by time.
func (c *Cache) Series(id string) ([]model.Point, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return nil, false
}
points, ok := c.snapshot.Series[id]
return points, ok
}
// Context returns the current weather/holiday context.
func (c *Cache) Context() model.ContextData {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return model.ContextData{}
}
return c.snapshot.Context
}
// Versions returns data, model and context version strings.
func (c *Cache) Versions() (data, modelVersion, contextVersion string) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return "", "", ""
}
return c.snapshot.DataVersion, c.snapshot.ModelVersion, c.snapshot.ContextVersion
}
// Len returns the number of sensors in the current snapshot.
func (c *Cache) Len() int {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return 0
}
return len(c.snapshot.Sensors)
}
func cloneStatus(s model.CacheStatus) model.CacheStatus {
out := s
if s.LastSuccessAt != nil {
v := *s.LastSuccessAt
out.LastSuccessAt = &v
}
if s.LastError != nil {
v := *s.LastError
out.LastError = &v
}
if s.NextRetryAt != nil {
v := *s.NextRetryAt
out.NextRetryAt = &v
}
return out
}
func timePtr(t time.Time) *time.Time { return &t }
+295
View File
@@ -0,0 +1,295 @@
package cache
import (
"context"
"errors"
"sync"
"testing"
"time"
"pulsecast/api/internal/model"
)
type stubFetcher struct {
mu sync.Mutex
calls int
payload model.Bootstrap
err error
fn func(int) (model.Bootstrap, error)
}
func (s *stubFetcher) Bootstrap(context.Context) (model.Bootstrap, error) {
s.mu.Lock()
s.calls++
call := s.calls
payload, err, fn := s.payload, s.err, s.fn
s.mu.Unlock()
if fn != nil {
return fn(call)
}
return payload, err
}
func (s *stubFetcher) set(payload model.Bootstrap, err error) {
s.mu.Lock()
s.payload, s.err = payload, err
s.mu.Unlock()
}
func (s *stubFetcher) callCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.calls
}
func fixturePayload() model.Bootstrap {
t0 := time.Date(2026, 11, 14, 3, 0, 0, 0, time.UTC)
return model.Bootstrap{
DataVersion: "2026-11-07T23:51:00Z",
ModelVersion: "histgb-2026-11-14",
ContextVersion: "2026-11-14T03:00:00Z",
Sensors: []model.Sensor{
{ID: "hn-zebra-01", Name: "Zebra", Lat: 49.1, Lon: 9.2, Mean: 1, P95: 2},
{ID: "hn-alpha-01", Name: "Alpha", Lat: 49.1, Lon: 9.2, Mean: 3, P95: 4},
},
Series: map[string][]model.Point{
"hn-alpha-01": {
{T: t0.Add(2 * time.Hour), V: 3},
{T: t0, V: 1},
{T: t0.Add(time.Hour), V: 2},
},
},
Context: model.ContextData{
Weather: model.Weather{
Now: model.WeatherNow{T: t0, TempC: 8.9, HumidityPct: 79},
Forecast: []model.WeatherPoint{{T: t0.Add(time.Hour), TempC: 8.4}},
},
Holidays: []model.Holiday{{Date: "2026-11-01", Name: "Allerheiligen"}},
UpdatedAt: t0,
},
Meta: model.BootstrapMeta{
Coverage: model.Coverage{Sensors: 8, From: "2026-06-12T00:00:00Z", To: "2026-11-07T23:51:00Z", CadenceMin: 15},
Metrics: model.Metrics{Model: "weather", MAE: 3.36, WAPE: 27.8, R2: 0.779},
Limits: []string{"estimated devices, not people"},
},
}
}
func TestRetryDelaySchedule(t *testing.T) {
c := New(&stubFetcher{}, Options{})
want := []time.Duration{1 * time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second, 30 * time.Second, 30 * time.Second}
for i, expected := range want {
if got := c.retryDelay(i + 1); got != expected {
t.Errorf("retryDelay(%d)=%s, want %s", i+1, got, expected)
}
}
for _, failures := range []int{0, -1} {
if got := c.retryDelay(failures); got != time.Second {
t.Errorf("retryDelay(%d)=%s, want 1s", failures, got)
}
}
}
func TestRefreshLoadsSnapshotAndStatus(t *testing.T) {
fetcher := &stubFetcher{payload: fixturePayload()}
c := New(fetcher, Options{RefreshInterval: time.Minute, Timeout: time.Second})
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
c.now = func() time.Time { return fixed }
if err := c.Refresh(context.Background()); err != nil {
t.Fatalf("Refresh: %v", err)
}
if !c.Loaded() || !c.Ready() || !c.Fresh() {
t.Fatalf("loaded=%v ready=%v fresh=%v", c.Loaded(), c.Ready(), c.Fresh())
}
data, modelOK := c.Readiness()
if !data || !modelOK {
t.Fatalf("readiness data=%v model=%v", data, modelOK)
}
if c.Len() != 2 {
t.Fatalf("Len=%d, want 2", c.Len())
}
sensors := c.Sensors()
if len(sensors) != 2 || sensors[0].ID != "hn-alpha-01" || sensors[1].ID != "hn-zebra-01" {
t.Fatalf("sensors not sorted: %+v", sensors)
}
if sensor, ok := c.Sensor("hn-zebra-01"); !ok || sensor.Name != "Zebra" {
t.Fatalf("Sensor: ok=%v sensor=%+v", ok, sensor)
}
if _, ok := c.Sensor("nope"); ok {
t.Fatal("unexpected sensor nope")
}
series, ok := c.Series("hn-alpha-01")
if !ok || len(series) != 3 || !series[0].T.Before(series[1].T) || !series[1].T.Before(series[2].T) {
t.Fatalf("series not sorted: ok=%v points=%+v", ok, series)
}
if ctx := c.Context(); ctx.UpdatedAt.IsZero() || len(ctx.Holidays) != 1 || len(ctx.Weather.Forecast) != 1 {
t.Fatalf("context=%+v", ctx)
}
dataVersion, modelVersion, contextVersion := c.Versions()
if dataVersion != "2026-11-07T23:51:00Z" || modelVersion != "histgb-2026-11-14" || contextVersion != "2026-11-14T03:00:00Z" {
t.Fatalf("versions=%q %q %q", dataVersion, modelVersion, contextVersion)
}
meta := c.Meta()
if meta.Coverage.Sensors != 8 || meta.Metrics.MAE != 3.36 || len(meta.Limits) != 1 {
t.Fatalf("meta=%+v", meta)
}
if meta.Cache.Source != Source || meta.Cache.LastSuccessAt == nil ||
!meta.Cache.LastSuccessAt.Equal(fixed) || meta.Cache.LastError != nil ||
meta.Cache.NextRetryAt != nil || meta.Cache.Failures != 0 {
t.Fatalf("cache status=%+v", meta.Cache)
}
}
func TestRefreshFailureKeepsLastGoodAndCountsBackoff(t *testing.T) {
fetcher := &stubFetcher{payload: fixturePayload()}
c := New(fetcher, Options{RefreshInterval: time.Minute, Timeout: time.Second})
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
c.now = func() time.Time { return fixed }
if err := c.Refresh(context.Background()); err != nil {
t.Fatalf("prime: %v", err)
}
primed := c.Sensors()
fetcher.set(model.Bootstrap{}, errors.New("connection refused"))
if err := c.Refresh(context.Background()); err == nil {
t.Fatal("expected refresh failure")
}
if !c.Ready() || c.Fresh() {
t.Fatalf("ready=%v fresh=%v, want ready and not fresh", c.Ready(), c.Fresh())
}
if got := c.Sensors(); len(got) != len(primed) || got[0].ID != primed[0].ID {
t.Fatalf("last good snapshot lost: %+v", got)
}
status := c.Status()
if status.Failures != 1 || status.LastError == nil || *status.LastError != "connection refused" {
t.Fatalf("status=%+v", status)
}
if status.LastSuccessAt == nil || !status.LastSuccessAt.Equal(fixed) {
t.Fatalf("last success=%v", status.LastSuccessAt)
}
if status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(time.Second)) {
t.Fatalf("next retry=%v, want %s", status.NextRetryAt, fixed.Add(time.Second))
}
if err := c.Refresh(context.Background()); err == nil {
t.Fatal("expected second refresh failure")
}
status = c.Status()
if status.Failures != 2 || status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(2*time.Second)) {
t.Fatalf("second failure status=%+v", status)
}
fetcher.set(fixturePayload(), nil)
if err := c.Refresh(context.Background()); err != nil {
t.Fatalf("recovery: %v", err)
}
status = c.Status()
if status.Failures != 0 || status.LastError != nil || status.NextRetryAt != nil || !c.Fresh() {
t.Fatalf("recovered status=%+v fresh=%v", status, c.Fresh())
}
if status.LastSuccessAt == nil || !status.LastSuccessAt.Equal(fixed) {
t.Fatalf("last success after recovery=%v", status.LastSuccessAt)
}
}
func TestRefreshRejectsIncompleteSnapshot(t *testing.T) {
for name, mutate := range map[string]func(*model.Bootstrap){
"no sensors": func(b *model.Bootstrap) { b.Sensors = nil },
"no data version": func(b *model.Bootstrap) { b.DataVersion = "" },
"no model version": func(b *model.Bootstrap) { b.ModelVersion = "" },
"no context version": func(b *model.Bootstrap) { b.ContextVersion = "" },
"no context time": func(b *model.Bootstrap) { b.Context.UpdatedAt = time.Time{} },
} {
t.Run(name, func(t *testing.T) {
payload := fixturePayload()
mutate(&payload)
c := New(&stubFetcher{payload: payload}, Options{Timeout: time.Second})
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
c.now = func() time.Time { return fixed }
if err := c.Refresh(context.Background()); err == nil {
t.Fatal("expected validation error")
}
if c.Loaded() || c.Ready() {
t.Fatalf("loaded=%v ready=%v after invalid snapshot", c.Loaded(), c.Ready())
}
if status := c.Status(); status.Failures != 1 || status.LastError == nil ||
status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(time.Second)) {
t.Fatalf("status=%+v", status)
}
})
}
}
func TestRunRetriesWithBackoffThenLoads(t *testing.T) {
fetcher := &stubFetcher{fn: func(call int) (model.Bootstrap, error) {
if call <= 2 {
return model.Bootstrap{}, errors.New("not ready")
}
return fixturePayload(), nil
}}
c := New(fetcher, Options{RefreshInterval: 10 * time.Millisecond, Timeout: time.Second})
c.retry = []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 20 * time.Millisecond}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan struct{})
go func() {
c.Run(ctx)
close(done)
}()
deadline := time.Now().Add(2 * time.Second)
for !c.Loaded() {
if time.Now().After(deadline) {
t.Fatalf("cache did not load after retries; calls=%d status=%+v", fetcher.callCount(), c.Status())
}
time.Sleep(5 * time.Millisecond)
}
if got := fetcher.callCount(); got < 3 {
t.Fatalf("calls=%d, want at least 3", got)
}
if status := c.Status(); status.Failures != 0 || status.LastError != nil {
t.Fatalf("status after recovery=%+v", status)
}
cancel()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("Run did not stop after context cancellation")
}
}
func TestRunStopsWhenContextCancelled(t *testing.T) {
fetcher := &stubFetcher{err: errors.New("down")}
c := New(fetcher, Options{RefreshInterval: time.Hour, Timeout: time.Second})
c.retry = []time.Duration{time.Hour}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
c.Run(ctx)
close(done)
}()
deadline := time.Now().Add(time.Second)
for fetcher.callCount() == 0 {
if time.Now().After(deadline) {
t.Fatal("Run never attempted the first fetch")
}
time.Sleep(time.Millisecond)
}
cancel()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("Run did not stop after context cancellation")
}
if status := c.Status(); status.Failures != 1 || status.NextRetryAt == nil {
t.Fatalf("status=%+v", status)
}
}