feat: runtime bootstrap snapshot cache with backoff
This commit is contained in:
1 parent
08f60a14d1
commit
aa6bb23e24
19 files changed
+1400
-384
No files matched your search
Vendored
+358
@@ -0,0 +1,358 @@
|
||||
// Package cache keeps the FastAPI /bootstrap snapshot in memory and refreshes
|
||||
// it in the background. A failed refresh never discards the last good snapshot;
|
||||
// the next attempt follows the contract backoff 1/2/4/8/16/30 s (cap).
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pulsecast/api/internal/model"
|
||||
)
|
||||
|
||||
// Source is the only snapshot producer: the FastAPI /bootstrap endpoint.
|
||||
const Source = "fastapi"
|
||||
|
||||
// DefaultRefreshInterval is how often a healthy snapshot is refreshed.
|
||||
const DefaultRefreshInterval = 60 * time.Second
|
||||
|
||||
// DefaultTimeout bounds one /bootstrap request.
|
||||
const DefaultTimeout = 20 * time.Second
|
||||
|
||||
// DefaultRetryDelays is the contract backoff schedule. The last value repeats.
|
||||
var DefaultRetryDelays = []time.Duration{
|
||||
1 * time.Second,
|
||||
2 * time.Second,
|
||||
4 * time.Second,
|
||||
8 * time.Second,
|
||||
16 * time.Second,
|
||||
30 * time.Second,
|
||||
}
|
||||
|
||||
// Fetcher is the FastAPI bootstrap surface. *ml.Client implements it.
|
||||
type Fetcher interface {
|
||||
Bootstrap(ctx context.Context) (model.Bootstrap, error)
|
||||
}
|
||||
|
||||
// Options configures refresh behaviour. Zero values use the defaults.
|
||||
type Options struct {
|
||||
RefreshInterval time.Duration
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// snapshot is one immutable /bootstrap payload with its lookup indexes.
|
||||
type snapshot struct {
|
||||
DataVersion string
|
||||
ModelVersion string
|
||||
ContextVersion string
|
||||
Sensors []model.Sensor
|
||||
Series map[string][]model.Point
|
||||
Context model.ContextData
|
||||
Coverage model.Coverage
|
||||
Metrics model.Metrics
|
||||
Limits []string
|
||||
|
||||
sensorBy map[string]model.Sensor
|
||||
}
|
||||
|
||||
// Cache holds the current snapshot and its refresh status. All methods are
|
||||
// safe for concurrent use.
|
||||
type Cache struct {
|
||||
fetcher Fetcher
|
||||
refresh time.Duration
|
||||
timeout time.Duration
|
||||
retry []time.Duration
|
||||
now func() time.Time
|
||||
mu sync.RWMutex
|
||||
snapshot *snapshot
|
||||
loaded bool
|
||||
status model.CacheStatus
|
||||
}
|
||||
|
||||
// New returns an empty cache. Call Run (or Refresh) to load the first snapshot.
|
||||
func New(fetcher Fetcher, opts Options) *Cache {
|
||||
if opts.RefreshInterval <= 0 {
|
||||
opts.RefreshInterval = DefaultRefreshInterval
|
||||
}
|
||||
if opts.Timeout <= 0 {
|
||||
opts.Timeout = DefaultTimeout
|
||||
}
|
||||
return &Cache{
|
||||
fetcher: fetcher,
|
||||
refresh: opts.RefreshInterval,
|
||||
timeout: opts.Timeout,
|
||||
retry: append([]time.Duration(nil), DefaultRetryDelays...),
|
||||
now: time.Now,
|
||||
status: model.CacheStatus{Source: Source},
|
||||
}
|
||||
}
|
||||
|
||||
// Run loads the snapshot immediately, then refreshes it. Failures keep the last
|
||||
// good snapshot and retry with the contract backoff. Run returns when ctx ends.
|
||||
func (c *Cache) Run(ctx context.Context) {
|
||||
recovering := false
|
||||
for {
|
||||
wasLoaded := c.Loaded()
|
||||
err := c.Refresh(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var delay time.Duration
|
||||
if err != nil {
|
||||
recovering = true
|
||||
status := c.Status()
|
||||
delay = c.retryDelay(status.Failures)
|
||||
slog.Warn("bootstrap refresh failed; keeping last good snapshot",
|
||||
"error", err, "failures", status.Failures, "retry_in", delay)
|
||||
} else {
|
||||
delay = c.refresh
|
||||
switch {
|
||||
case !wasLoaded:
|
||||
slog.Info("bootstrap snapshot loaded", "sensors", c.Len())
|
||||
case recovering:
|
||||
slog.Info("bootstrap refresh recovered", "sensors", c.Len())
|
||||
}
|
||||
recovering = false
|
||||
}
|
||||
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh performs one fetch. A decode/validation failure counts as a failed
|
||||
// attempt and leaves the previous snapshot untouched.
|
||||
func (c *Cache) Refresh(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
||||
defer cancel()
|
||||
|
||||
payload, err := c.fetcher.Bootstrap(ctx)
|
||||
now := c.now().UTC().Truncate(time.Second)
|
||||
if err == nil {
|
||||
var snapshot *snapshot
|
||||
snapshot, err = newSnapshot(payload)
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
c.snapshot = snapshot
|
||||
c.loaded = true
|
||||
c.status.LastSuccessAt = timePtr(now)
|
||||
c.status.LastError = nil
|
||||
c.status.NextRetryAt = nil
|
||||
c.status.Failures = 0
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
c.status.Failures++
|
||||
message := err.Error()
|
||||
c.status.LastError = &message
|
||||
next := now.Add(c.retryDelay(c.status.Failures))
|
||||
c.status.NextRetryAt = &next
|
||||
c.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cache) retryDelay(failures int) time.Duration {
|
||||
if failures < 1 {
|
||||
failures = 1
|
||||
}
|
||||
if failures > len(c.retry) {
|
||||
failures = len(c.retry)
|
||||
}
|
||||
return c.retry[failures-1]
|
||||
}
|
||||
|
||||
func newSnapshot(payload model.Bootstrap) (*snapshot, error) {
|
||||
switch {
|
||||
case len(payload.Sensors) == 0:
|
||||
return nil, fmt.Errorf("bootstrap snapshot: no sensors")
|
||||
case payload.DataVersion == "":
|
||||
return nil, fmt.Errorf("bootstrap snapshot: missing data_version")
|
||||
case payload.ModelVersion == "":
|
||||
return nil, fmt.Errorf("bootstrap snapshot: missing model_version")
|
||||
case payload.ContextVersion == "":
|
||||
return nil, fmt.Errorf("bootstrap snapshot: missing context_version")
|
||||
case payload.Context.UpdatedAt.IsZero():
|
||||
return nil, fmt.Errorf("bootstrap snapshot: missing context.updated_at")
|
||||
}
|
||||
|
||||
snapshot := &snapshot{
|
||||
DataVersion: payload.DataVersion,
|
||||
ModelVersion: payload.ModelVersion,
|
||||
ContextVersion: payload.ContextVersion,
|
||||
Sensors: append([]model.Sensor(nil), payload.Sensors...),
|
||||
Series: make(map[string][]model.Point, len(payload.Series)),
|
||||
Context: payload.Context,
|
||||
Coverage: payload.Meta.Coverage,
|
||||
Metrics: payload.Meta.Metrics,
|
||||
Limits: append([]string(nil), payload.Meta.Limits...),
|
||||
sensorBy: make(map[string]model.Sensor, len(payload.Sensors)),
|
||||
}
|
||||
sort.Slice(snapshot.Sensors, func(i, j int) bool { return snapshot.Sensors[i].ID < snapshot.Sensors[j].ID })
|
||||
for _, sensor := range snapshot.Sensors {
|
||||
snapshot.sensorBy[sensor.ID] = sensor
|
||||
}
|
||||
for sensorID, points := range payload.Series {
|
||||
copied := append([]model.Point(nil), points...)
|
||||
sort.Slice(copied, func(i, j int) bool { return copied[i].T.Before(copied[j].T) })
|
||||
snapshot.Series[sensorID] = copied
|
||||
}
|
||||
snapshot.Context.Holidays = append([]model.Holiday(nil), payload.Context.Holidays...)
|
||||
snapshot.Context.Weather.Forecast = append([]model.WeatherPoint(nil), payload.Context.Weather.Forecast...)
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// Loaded reports whether a snapshot has ever been accepted.
|
||||
func (c *Cache) Loaded() bool {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.loaded
|
||||
}
|
||||
|
||||
// Ready reports whether a complete snapshot is available. FastAPI health is
|
||||
// deliberately not part of this check: the cache itself is the safety net.
|
||||
func (c *Cache) Ready() bool {
|
||||
data, modelOK := c.Readiness()
|
||||
return data && modelOK
|
||||
}
|
||||
|
||||
// Readiness splits readiness into the data and model signals of /api/readyz.
|
||||
func (c *Cache) Readiness() (data, modelOK bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if !c.loaded || c.snapshot == nil {
|
||||
return false, false
|
||||
}
|
||||
data = len(c.snapshot.Sensors) > 0 && !c.snapshot.Context.UpdatedAt.IsZero()
|
||||
modelOK = c.snapshot.ModelVersion != ""
|
||||
return data, modelOK
|
||||
}
|
||||
|
||||
// Fresh reports whether the most recent refresh attempt succeeded. It is false
|
||||
// before the first snapshot and after a failed refresh.
|
||||
func (c *Cache) Fresh() bool {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.loaded && c.status.Failures == 0
|
||||
}
|
||||
|
||||
// Status returns a copy of the cache status.
|
||||
func (c *Cache) Status() model.CacheStatus {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return cloneStatus(c.status)
|
||||
}
|
||||
|
||||
// Meta combines the snapshot metadata with the live cache status.
|
||||
func (c *Cache) Meta() model.Meta {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
meta := model.Meta{Cache: cloneStatus(c.status)}
|
||||
if c.snapshot == nil {
|
||||
return meta
|
||||
}
|
||||
meta.DataVersion = c.snapshot.DataVersion
|
||||
meta.ModelVersion = c.snapshot.ModelVersion
|
||||
meta.ContextVersion = c.snapshot.ContextVersion
|
||||
meta.Coverage = c.snapshot.Coverage
|
||||
meta.Metrics = c.snapshot.Metrics
|
||||
meta.Limits = append([]string(nil), c.snapshot.Limits...)
|
||||
return meta
|
||||
}
|
||||
|
||||
// Sensors returns the sensors sorted by id.
|
||||
func (c *Cache) Sensors() []model.Sensor {
|
||||
c.mu.RLock()
|
||||
snapshot := c.snapshot
|
||||
c.mu.RUnlock()
|
||||
if snapshot == nil {
|
||||
return []model.Sensor{}
|
||||
}
|
||||
out := make([]model.Sensor, len(snapshot.Sensors))
|
||||
copy(out, snapshot.Sensors)
|
||||
return out
|
||||
}
|
||||
|
||||
// Sensor looks up one sensor.
|
||||
func (c *Cache) Sensor(id string) (model.Sensor, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return model.Sensor{}, false
|
||||
}
|
||||
sensor, ok := c.snapshot.sensorBy[id]
|
||||
return sensor, ok
|
||||
}
|
||||
|
||||
// Series returns the immutable series for a sensor, sorted by time.
|
||||
func (c *Cache) Series(id string) ([]model.Point, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return nil, false
|
||||
}
|
||||
points, ok := c.snapshot.Series[id]
|
||||
return points, ok
|
||||
}
|
||||
|
||||
// Context returns the current weather/holiday context.
|
||||
func (c *Cache) Context() model.ContextData {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return model.ContextData{}
|
||||
}
|
||||
return c.snapshot.Context
|
||||
}
|
||||
|
||||
// Versions returns data, model and context version strings.
|
||||
func (c *Cache) Versions() (data, modelVersion, contextVersion string) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return "", "", ""
|
||||
}
|
||||
return c.snapshot.DataVersion, c.snapshot.ModelVersion, c.snapshot.ContextVersion
|
||||
}
|
||||
|
||||
// Len returns the number of sensors in the current snapshot.
|
||||
func (c *Cache) Len() int {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return 0
|
||||
}
|
||||
return len(c.snapshot.Sensors)
|
||||
}
|
||||
|
||||
func cloneStatus(s model.CacheStatus) model.CacheStatus {
|
||||
out := s
|
||||
if s.LastSuccessAt != nil {
|
||||
v := *s.LastSuccessAt
|
||||
out.LastSuccessAt = &v
|
||||
}
|
||||
if s.LastError != nil {
|
||||
v := *s.LastError
|
||||
out.LastError = &v
|
||||
}
|
||||
if s.NextRetryAt != nil {
|
||||
v := *s.NextRetryAt
|
||||
out.NextRetryAt = &v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func timePtr(t time.Time) *time.Time { return &t }
|
||||
Vendored
+295
@@ -0,0 +1,295 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pulsecast/api/internal/model"
|
||||
)
|
||||
|
||||
type stubFetcher struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
payload model.Bootstrap
|
||||
err error
|
||||
fn func(int) (model.Bootstrap, error)
|
||||
}
|
||||
|
||||
func (s *stubFetcher) Bootstrap(context.Context) (model.Bootstrap, error) {
|
||||
s.mu.Lock()
|
||||
s.calls++
|
||||
call := s.calls
|
||||
payload, err, fn := s.payload, s.err, s.fn
|
||||
s.mu.Unlock()
|
||||
if fn != nil {
|
||||
return fn(call)
|
||||
}
|
||||
return payload, err
|
||||
}
|
||||
|
||||
func (s *stubFetcher) set(payload model.Bootstrap, err error) {
|
||||
s.mu.Lock()
|
||||
s.payload, s.err = payload, err
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *stubFetcher) callCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.calls
|
||||
}
|
||||
|
||||
func fixturePayload() model.Bootstrap {
|
||||
t0 := time.Date(2026, 11, 14, 3, 0, 0, 0, time.UTC)
|
||||
return model.Bootstrap{
|
||||
DataVersion: "2026-11-07T23:51:00Z",
|
||||
ModelVersion: "histgb-2026-11-14",
|
||||
ContextVersion: "2026-11-14T03:00:00Z",
|
||||
Sensors: []model.Sensor{
|
||||
{ID: "hn-zebra-01", Name: "Zebra", Lat: 49.1, Lon: 9.2, Mean: 1, P95: 2},
|
||||
{ID: "hn-alpha-01", Name: "Alpha", Lat: 49.1, Lon: 9.2, Mean: 3, P95: 4},
|
||||
},
|
||||
Series: map[string][]model.Point{
|
||||
"hn-alpha-01": {
|
||||
{T: t0.Add(2 * time.Hour), V: 3},
|
||||
{T: t0, V: 1},
|
||||
{T: t0.Add(time.Hour), V: 2},
|
||||
},
|
||||
},
|
||||
Context: model.ContextData{
|
||||
Weather: model.Weather{
|
||||
Now: model.WeatherNow{T: t0, TempC: 8.9, HumidityPct: 79},
|
||||
Forecast: []model.WeatherPoint{{T: t0.Add(time.Hour), TempC: 8.4}},
|
||||
},
|
||||
Holidays: []model.Holiday{{Date: "2026-11-01", Name: "Allerheiligen"}},
|
||||
UpdatedAt: t0,
|
||||
},
|
||||
Meta: model.BootstrapMeta{
|
||||
Coverage: model.Coverage{Sensors: 8, From: "2026-06-12T00:00:00Z", To: "2026-11-07T23:51:00Z", CadenceMin: 15},
|
||||
Metrics: model.Metrics{Model: "weather", MAE: 3.36, WAPE: 27.8, R2: 0.779},
|
||||
Limits: []string{"estimated devices, not people"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryDelaySchedule(t *testing.T) {
|
||||
c := New(&stubFetcher{}, Options{})
|
||||
want := []time.Duration{1 * time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second, 30 * time.Second, 30 * time.Second}
|
||||
for i, expected := range want {
|
||||
if got := c.retryDelay(i + 1); got != expected {
|
||||
t.Errorf("retryDelay(%d)=%s, want %s", i+1, got, expected)
|
||||
}
|
||||
}
|
||||
for _, failures := range []int{0, -1} {
|
||||
if got := c.retryDelay(failures); got != time.Second {
|
||||
t.Errorf("retryDelay(%d)=%s, want 1s", failures, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshLoadsSnapshotAndStatus(t *testing.T) {
|
||||
fetcher := &stubFetcher{payload: fixturePayload()}
|
||||
c := New(fetcher, Options{RefreshInterval: time.Minute, Timeout: time.Second})
|
||||
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
|
||||
c.now = func() time.Time { return fixed }
|
||||
|
||||
if err := c.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("Refresh: %v", err)
|
||||
}
|
||||
if !c.Loaded() || !c.Ready() || !c.Fresh() {
|
||||
t.Fatalf("loaded=%v ready=%v fresh=%v", c.Loaded(), c.Ready(), c.Fresh())
|
||||
}
|
||||
data, modelOK := c.Readiness()
|
||||
if !data || !modelOK {
|
||||
t.Fatalf("readiness data=%v model=%v", data, modelOK)
|
||||
}
|
||||
if c.Len() != 2 {
|
||||
t.Fatalf("Len=%d, want 2", c.Len())
|
||||
}
|
||||
|
||||
sensors := c.Sensors()
|
||||
if len(sensors) != 2 || sensors[0].ID != "hn-alpha-01" || sensors[1].ID != "hn-zebra-01" {
|
||||
t.Fatalf("sensors not sorted: %+v", sensors)
|
||||
}
|
||||
if sensor, ok := c.Sensor("hn-zebra-01"); !ok || sensor.Name != "Zebra" {
|
||||
t.Fatalf("Sensor: ok=%v sensor=%+v", ok, sensor)
|
||||
}
|
||||
if _, ok := c.Sensor("nope"); ok {
|
||||
t.Fatal("unexpected sensor nope")
|
||||
}
|
||||
series, ok := c.Series("hn-alpha-01")
|
||||
if !ok || len(series) != 3 || !series[0].T.Before(series[1].T) || !series[1].T.Before(series[2].T) {
|
||||
t.Fatalf("series not sorted: ok=%v points=%+v", ok, series)
|
||||
}
|
||||
if ctx := c.Context(); ctx.UpdatedAt.IsZero() || len(ctx.Holidays) != 1 || len(ctx.Weather.Forecast) != 1 {
|
||||
t.Fatalf("context=%+v", ctx)
|
||||
}
|
||||
dataVersion, modelVersion, contextVersion := c.Versions()
|
||||
if dataVersion != "2026-11-07T23:51:00Z" || modelVersion != "histgb-2026-11-14" || contextVersion != "2026-11-14T03:00:00Z" {
|
||||
t.Fatalf("versions=%q %q %q", dataVersion, modelVersion, contextVersion)
|
||||
}
|
||||
|
||||
meta := c.Meta()
|
||||
if meta.Coverage.Sensors != 8 || meta.Metrics.MAE != 3.36 || len(meta.Limits) != 1 {
|
||||
t.Fatalf("meta=%+v", meta)
|
||||
}
|
||||
if meta.Cache.Source != Source || meta.Cache.LastSuccessAt == nil ||
|
||||
!meta.Cache.LastSuccessAt.Equal(fixed) || meta.Cache.LastError != nil ||
|
||||
meta.Cache.NextRetryAt != nil || meta.Cache.Failures != 0 {
|
||||
t.Fatalf("cache status=%+v", meta.Cache)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshFailureKeepsLastGoodAndCountsBackoff(t *testing.T) {
|
||||
fetcher := &stubFetcher{payload: fixturePayload()}
|
||||
c := New(fetcher, Options{RefreshInterval: time.Minute, Timeout: time.Second})
|
||||
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
|
||||
c.now = func() time.Time { return fixed }
|
||||
|
||||
if err := c.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("prime: %v", err)
|
||||
}
|
||||
primed := c.Sensors()
|
||||
|
||||
fetcher.set(model.Bootstrap{}, errors.New("connection refused"))
|
||||
if err := c.Refresh(context.Background()); err == nil {
|
||||
t.Fatal("expected refresh failure")
|
||||
}
|
||||
if !c.Ready() || c.Fresh() {
|
||||
t.Fatalf("ready=%v fresh=%v, want ready and not fresh", c.Ready(), c.Fresh())
|
||||
}
|
||||
if got := c.Sensors(); len(got) != len(primed) || got[0].ID != primed[0].ID {
|
||||
t.Fatalf("last good snapshot lost: %+v", got)
|
||||
}
|
||||
status := c.Status()
|
||||
if status.Failures != 1 || status.LastError == nil || *status.LastError != "connection refused" {
|
||||
t.Fatalf("status=%+v", status)
|
||||
}
|
||||
if status.LastSuccessAt == nil || !status.LastSuccessAt.Equal(fixed) {
|
||||
t.Fatalf("last success=%v", status.LastSuccessAt)
|
||||
}
|
||||
if status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(time.Second)) {
|
||||
t.Fatalf("next retry=%v, want %s", status.NextRetryAt, fixed.Add(time.Second))
|
||||
}
|
||||
|
||||
if err := c.Refresh(context.Background()); err == nil {
|
||||
t.Fatal("expected second refresh failure")
|
||||
}
|
||||
status = c.Status()
|
||||
if status.Failures != 2 || status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(2*time.Second)) {
|
||||
t.Fatalf("second failure status=%+v", status)
|
||||
}
|
||||
|
||||
fetcher.set(fixturePayload(), nil)
|
||||
if err := c.Refresh(context.Background()); err != nil {
|
||||
t.Fatalf("recovery: %v", err)
|
||||
}
|
||||
status = c.Status()
|
||||
if status.Failures != 0 || status.LastError != nil || status.NextRetryAt != nil || !c.Fresh() {
|
||||
t.Fatalf("recovered status=%+v fresh=%v", status, c.Fresh())
|
||||
}
|
||||
if status.LastSuccessAt == nil || !status.LastSuccessAt.Equal(fixed) {
|
||||
t.Fatalf("last success after recovery=%v", status.LastSuccessAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshRejectsIncompleteSnapshot(t *testing.T) {
|
||||
for name, mutate := range map[string]func(*model.Bootstrap){
|
||||
"no sensors": func(b *model.Bootstrap) { b.Sensors = nil },
|
||||
"no data version": func(b *model.Bootstrap) { b.DataVersion = "" },
|
||||
"no model version": func(b *model.Bootstrap) { b.ModelVersion = "" },
|
||||
"no context version": func(b *model.Bootstrap) { b.ContextVersion = "" },
|
||||
"no context time": func(b *model.Bootstrap) { b.Context.UpdatedAt = time.Time{} },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
payload := fixturePayload()
|
||||
mutate(&payload)
|
||||
c := New(&stubFetcher{payload: payload}, Options{Timeout: time.Second})
|
||||
fixed := time.Date(2026, 11, 14, 4, 0, 0, 0, time.UTC)
|
||||
c.now = func() time.Time { return fixed }
|
||||
if err := c.Refresh(context.Background()); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
if c.Loaded() || c.Ready() {
|
||||
t.Fatalf("loaded=%v ready=%v after invalid snapshot", c.Loaded(), c.Ready())
|
||||
}
|
||||
if status := c.Status(); status.Failures != 1 || status.LastError == nil ||
|
||||
status.NextRetryAt == nil || !status.NextRetryAt.Equal(fixed.Add(time.Second)) {
|
||||
t.Fatalf("status=%+v", status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRetriesWithBackoffThenLoads(t *testing.T) {
|
||||
fetcher := &stubFetcher{fn: func(call int) (model.Bootstrap, error) {
|
||||
if call <= 2 {
|
||||
return model.Bootstrap{}, errors.New("not ready")
|
||||
}
|
||||
return fixturePayload(), nil
|
||||
}}
|
||||
c := New(fetcher, Options{RefreshInterval: 10 * time.Millisecond, Timeout: time.Second})
|
||||
c.retry = []time.Duration{5 * time.Millisecond, 10 * time.Millisecond, 20 * time.Millisecond}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
c.Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for !c.Loaded() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("cache did not load after retries; calls=%d status=%+v", fetcher.callCount(), c.Status())
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if got := fetcher.callCount(); got < 3 {
|
||||
t.Fatalf("calls=%d, want at least 3", got)
|
||||
}
|
||||
if status := c.Status(); status.Failures != 0 || status.LastError != nil {
|
||||
t.Fatalf("status after recovery=%+v", status)
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Run did not stop after context cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsWhenContextCancelled(t *testing.T) {
|
||||
fetcher := &stubFetcher{err: errors.New("down")}
|
||||
c := New(fetcher, Options{RefreshInterval: time.Hour, Timeout: time.Second})
|
||||
c.retry = []time.Duration{time.Hour}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
c.Run(ctx)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for fetcher.callCount() == 0 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("Run never attempted the first fetch")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Run did not stop after context cancellation")
|
||||
}
|
||||
if status := c.Status(); status.Failures != 1 || status.NextRetryAt == nil {
|
||||
t.Fatalf("status=%+v", status)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user