feat: runtime bootstrap snapshot cache with backoff
This commit is contained in:
1 parent
08f60a14d1
commit
aa6bb23e24
19 files changed
+1400
-384
No files matched your search
Vendored
+295
@@ -0,0 +1,295 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pulsecast/api/internal/model"
|
||||
)
|
||||
|
||||
type stubFetcher struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
payload model.Bootstrap
|
||||
err error
|
||||
fn func(int) (model.Bootstrap, error)
|
||||
}
|
||||
|
||||
func (s *stubFetcher) Bootstrap(context.Context) (model.Bootstrap, error) {
|
||||
s.mu.Lock()
|
||||
s.calls++
|
||||
call := s.calls
|
||||
payload, err, fn := s.payload, s.err, s.fn
|
||||
s.mu.Unlock()
|
||||
if fn != nil {
|
||||
return fn(call)
|
||||
}
|
||||
return payload, err
|
||||
}
|
||||
|
||||
func (s *stubFetcher) set(payload model.Bootstrap, err error) {
|
||||
s.mu.Lock()
|
||||
s.payload, s.err = payload, err
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *stubFetcher) callCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.calls
|
||||
}
|
||||
|
||||
func fixturePayload() model.Bootstrap {
|
||||
t0 := time.Date(2026, 11, 14, 3, 0, 0, 0, time.UTC)
|
||||
return model.Bootstrap{
|
||||
DataVersion: "2026-11-07T23:51:00Z",
|
||||
ModelVersion: "histgb-2026-11-14",
|
||||
ContextVersion: "2026-11-14T03:00:00Z",
|
||||
Sensors: []model.Sensor{
|
||||
{ID: "hn-zebra-01", Name: "Zebra", Lat: 49.1, Lon: 9.2, Mean: 1, P95: 2},
|
||||
{ID: "hn-alpha-01", Name: "Alpha", Lat: 49.1, Lon: 9.2, Mean: 3, P95: 4},
|
||||
},
|
||||
Series: map[string][]model.Point{
|
||||
"hn-alpha-01": {
|
||||
{T: t0.Add(2 * time.Hour), V: 3},
|
||||
{T: t0, V: 1},
|
||||
{T: t0.Add(time.Hour), V: 2},
|
||||
},
|
||||
},
|
||||
Context: model.ContextData{
|
||||
Weather: model.Weather{
|
||||
Now: model.WeatherNow{T: t0, TempC: 8.9, HumidityPct: 79},
|
||||
Forecast: []model.WeatherPoint{{T: t0.Add(time.Hour), TempC: 8.4}},
|
||||
},
|
||||
Holidays: []model.Holiday{{Date: "2026-11-01", Name: "Allerheiligen"}},
|
||||
UpdatedAt: t0,
|
||||
},
|
||||
Meta: model.BootstrapMeta{
|
||||
Coverage: model.Coverage{Sensors: 8, From: "2026-06-12T00:00:00Z", To: "2026-11-07T23:51:00Z", CadenceMin: 15},
|
||||
Metrics: model.Metrics{Model: "weather", MAE: 3.36, WAPE: 27.8, R2: 0.779},
|
||||
Limits: []string{"estimated devices, not people"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryDelaySchedule(t *testing.T) {
|
||||
c := New(&stubFetcher{}, Options{})
|
||||
want := []time.Duration{1 * time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second, 30 * time.Second, 30 * time.Second}
|
||||
for i, expected := range want {
|
||||
if got := c.retryDelay(i + 1); got != expected {
|
||||
t.Errorf("retryDelay(%d)=%s, want %s", i+1, got, expected)
|
||||
}
|
||||
}
|
||||
for _, failures := range []int{0, -1} {
|
||||
if got := c.retryDelay(failures); got != time.Second {
|
||||
t.Errorf("retryDelay(%d)=%s, want 1s", failures, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshLoadsSnapshotAndStatus(t *testing.T) {
|
||||
fetcher := &stubFetcher{payload: fixturePayload()}
|
||||
c := New(fetcher, Options{RefreshInterval: time.Minute, Timeout: time.Second})
|
||||
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
|
||||
c.now = func() time.Time { return fixed }
|
||||
|
||||
if err := c.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("Refresh: %v", err)
|
||||
}
|
||||
if !c.Loaded() || !c.Ready() || !c.Fresh() {
|
||||
t.Fatalf("loaded=%v ready=%v fresh=%v", c.Loaded(), c.Ready(), c.Fresh())
|
||||
}
|
||||
data, modelOK := c.Readiness()
|
||||
if !data || !modelOK {
|
||||
t.Fatalf("readiness data=%v model=%v", data, modelOK)
|
||||
}
|
||||
if c.Len() != 2 {
|
||||
t.Fatalf("Len=%d, want 2", c.Len())
|
||||
}
|
||||
|
||||
sensors := c.Sensors()
|
||||
if len(sensors) != 2 || sensors[0].ID != "hn-alpha-01" || sensors[1].ID != "hn-zebra-01" {
|
||||
t.Fatalf("sensors not sorted: %+v", sensors)
|
||||
}
|
||||
if sensor, ok := c.Sensor("hn-zebra-01"); !ok || sensor.Name != "Zebra" {
|
||||
t.Fatalf("Sensor: ok=%v sensor=%+v", ok, sensor)
|
||||
}
|
||||
if _, ok := c.Sensor("nope"); ok {
|
||||
t.Fatal("unexpected sensor nope")
|
||||
}
|
||||
series, ok := c.Series("hn-alpha-01")
|
||||
if !ok || len(series) != 3 || !series[0].T.Before(series[1].T) || !series[1].T.Before(series[2].T) {
|
||||
t.Fatalf("series not sorted: ok=%v points=%+v", ok, series)
|
||||
}
|
||||
if ctx := c.Context(); ctx.UpdatedAt.IsZero() || len(ctx.Holidays) != 1 || len(ctx.Weather.Forecast) != 1 {
|
||||
t.Fatalf("context=%+v", ctx)
|
||||
}
|
||||
dataVersion, modelVersion, contextVersion := c.Versions()
|
||||
if dataVersion != "2026-11-07T23:51:00Z" || modelVersion != "histgb-2026-11-14" || contextVersion != "2026-11-14T03:00:00Z" {
|
||||
t.Fatalf("versions=%q %q %q", dataVersion, modelVersion, contextVersion)
|
||||
}
|
||||
|
||||
meta := c.Meta()
|
||||
if meta.Coverage.Sensors != 8 || meta.Metrics.MAE != 3.36 || len(meta.Limits) != 1 {
|
||||
t.Fatalf("meta=%+v", meta)
|
||||
}
|
||||
if meta.Cache.Source != Source || meta.Cache.LastSuccessAt == nil ||
|
||||
!meta.Cache.LastSuccessAt.Equal(fixed) || meta.Cache.LastError != nil ||
|
||||
meta.Cache.NextRetryAt != nil || meta.Cache.Failures != 0 {
|
||||
t.Fatalf("cache status=%+v", meta.Cache)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshFailureKeepsLastGoodAndCountsBackoff(t *testing.T) {
|
||||
fetcher := &stubFetcher{payload: fixturePayload()}
|
||||
c := New(fetcher, Options{RefreshInterval: time.Minute, Timeout: time.Second})
|
||||
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
|
||||
c.now = func() time.Time { return fixed }
|
||||
|
||||
if err := c.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("prime: %v", err)
|
||||
}
|
||||
primed := c.Sensors()
|
||||
|
||||
fetcher.set(model.Bootstrap{}, errors.New("connection refused"))
|
||||
if err := c.Refresh(context.Background()); err == nil {
|
||||
t.Fatal("expected refresh failure")
|
||||
}
|
||||
if !c.Ready() || c.Fresh() {
|
||||
t.Fatalf("ready=%v fresh=%v, want ready and not fresh", c.Ready(), c.Fresh())
|
||||
}
|
||||
if got := c.Sensors(); len(got) != len(primed) || got[0].ID != primed[0].ID {
|
||||
t.Fatalf("last good snapshot lost: %+v", got)
|
||||
}
|
||||
status := c.Status()
|
||||
if status.Failures != 1 || status.LastError == nil || *status.LastError != "connection refused" {
|
||||
t.Fatalf("status=%+v", status)
|
||||
}
|
||||
if status.LastSuccessAt == nil || !status.LastSuccessAt.Equal(fixed) {
|
||||
t.Fatalf("last success=%v", status.LastSuccessAt)
|
||||
}
|
||||
if status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(time.Second)) {
|
||||
t.Fatalf("next retry=%v, want %s", status.NextRetryAt, fixed.Add(time.Second))
|
||||
}
|
||||
|
||||
if err := c.Refresh(context.Background()); err == nil {
|
||||
t.Fatal("expected second refresh failure")
|
||||
}
|
||||
status = c.Status()
|
||||
if status.Failures != 2 || status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(2*time.Second)) {
|
||||
t.Fatalf("second failure status=%+v", status)
|
||||
}
|
||||
|
||||
fetcher.set(fixturePayload(), nil)
|
||||
if err := c.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("recovery: %v", err)
|
||||
}
|
||||
status = c.Status()
|
||||
if status.Failures != 0 || status.LastError != nil || status.NextRetryAt != nil || !c.Fresh() {
|
||||
t.Fatalf("recovered status=%+v fresh=%v", status, c.Fresh())
|
||||
}
|
||||
if status.LastSuccessAt == nil || !status.LastSuccessAt.Equal(fixed) {
|
||||
t.Fatalf("last success after recovery=%v", status.LastSuccessAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshRejectsIncompleteSnapshot(t *testing.T) {
|
||||
for name, mutate := range map[string]func(*model.Bootstrap){
|
||||
"no sensors": func(b *model.Bootstrap) { b.Sensors = nil },
|
||||
"no data version": func(b *model.Bootstrap) { b.DataVersion = "" },
|
||||
"no model version": func(b *model.Bootstrap) { b.ModelVersion = "" },
|
||||
"no context version": func(b *model.Bootstrap) { b.ContextVersion = "" },
|
||||
"no context time": func(b *model.Bootstrap) { b.Context.UpdatedAt = time.Time{} },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
payload := fixturePayload()
|
||||
mutate(&payload)
|
||||
c := New(&stubFetcher{payload: payload}, Options{Timeout: time.Second})
|
||||
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
|
||||
c.now = func() time.Time { return fixed }
|
||||
if err := c.Refresh(context.Background()); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
if c.Loaded() || c.Ready() {
|
||||
t.Fatalf("loaded=%v ready=%v after invalid snapshot", c.Loaded(), c.Ready())
|
||||
}
|
||||
if status := c.Status(); status.Failures != 1 || status.LastError == nil ||
|
||||
status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(time.Second)) {
|
||||
t.Fatalf("status=%+v", status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRetriesWithBackoffThenLoads(t *testing.T) {
|
||||
fetcher := &stubFetcher{fn: func(call int) (model.Bootstrap, error) {
|
||||
if call <= 2 {
|
||||
return model.Bootstrap{}, errors.New("not ready")
|
||||
}
|
||||
return fixturePayload(), nil
|
||||
}}
|
||||
c := New(fetcher, Options{RefreshInterval: 10 * time.Millisecond, Timeout: time.Second})
|
||||
c.retry = []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 20 * time.Millisecond}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
c.Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for !c.Loaded() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("cache did not load after retries; calls=%d status=%+v", fetcher.callCount(), c.Status())
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if got := fetcher.callCount(); got < 3 {
|
||||
t.Fatalf("calls=%d, want at least 3", got)
|
||||
}
|
||||
if status := c.Status(); status.Failures != 0 || status.LastError != nil {
|
||||
t.Fatalf("status after recovery=%+v", status)
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Run did not stop after context cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsWhenContextCancelled(t *testing.T) {
|
||||
fetcher := &stubFetcher{err: errors.New("down")}
|
||||
c := New(fetcher, Options{RefreshInterval: time.Hour, Timeout: time.Second})
|
||||
c.retry = []time.Duration{time.Hour}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
c.Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for fetcher.callCount() == 0 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("Run never attempted the first fetch")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Run did not stop after context cancellation")
|
||||
}
|
||||
if status := c.Status(); status.Failures != 1 || status.NextRetryAt == nil {
|
||||
t.Fatalf("status=%+v", status)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user