feat: runtime bootstrap snapshot cache with backoff

This commit is contained in:
Noel Sabia committed 2026-11-14 06:08:53 +01:00
1 parent 08f60a14d1
commit aa6bb23e24
19 files changed
+1400 -384

No files matched your search

+358
View File
@@ -0,0 +1,358 @@
// Package cache keeps the FastAPI /bootstrap snapshot in memory and refreshes
// it in the background. A failed refresh never discards the last good snapshot;
// the next attempt follows the contract backoff 1/2/4/8/16/30 s (cap).
package cache
import (
"context"
"fmt"
"log/slog"
"sort"
"sync"
"time"
"pulsecast/api/internal/model"
)
// Source is the only snapshot producer: the FastAPI /bootstrap endpoint.
const Source = "fastapi"
// DefaultRefreshInterval is how often a healthy snapshot is refreshed.
const DefaultRefreshInterval = 60 * time.Second
// DefaultTimeout bounds one /bootstrap request.
const DefaultTimeout = 20 * time.Second
// DefaultRetryDelays is the contract backoff schedule. The last value repeats.
var DefaultRetryDelays = []time.Duration{
1 * time.Second,
2 * time.Second,
4 * time.Second,
8 * time.Second,
16 * time.Second,
30 * time.Second,
}
// Fetcher is the FastAPI bootstrap surface. *ml.Client implements it.
type Fetcher interface {
Bootstrap(ctx context.Context) (model.Bootstrap, error)
}
// Options configures refresh behaviour. Zero values use the defaults.
type Options struct {
RefreshInterval time.Duration
Timeout time.Duration
}
// snapshot is one immutable /bootstrap payload with its lookup indexes.
type snapshot struct {
DataVersion string
ModelVersion string
ContextVersion string
Sensors []model.Sensor
Series map[string][]model.Point
Context model.ContextData
Coverage model.Coverage
Metrics model.Metrics
Limits []string
sensorBy map[string]model.Sensor
}
// Cache holds the current snapshot and its refresh status. All methods are
// safe for concurrent use.
type Cache struct {
fetcher Fetcher
refresh time.Duration
timeout time.Duration
retry []time.Duration
now func() time.Time
mu sync.RWMutex
snapshot *snapshot
loaded bool
status model.CacheStatus
}
// New returns an empty cache. Call Run (or Refresh) to load the first snapshot.
func New(fetcher Fetcher, opts Options) *Cache {
if opts.RefreshInterval <= 0 {
opts.RefreshInterval = DefaultRefreshInterval
}
if opts.Timeout <= 0 {
opts.Timeout = DefaultTimeout
}
return &Cache{
fetcher: fetcher,
refresh: opts.RefreshInterval,
timeout: opts.Timeout,
retry: append([]time.Duration(nil), DefaultRetryDelays...),
now: time.Now,
status: model.CacheStatus{Source: Source},
}
}
// Run loads the snapshot immediately, then refreshes it. Failures keep the last
// good snapshot and retry with the contract backoff. Run returns when ctx ends.
func (c *Cache) Run(ctx context.Context) {
recovering := false
for {
wasLoaded := c.Loaded()
err := c.Refresh(ctx)
if ctx.Err() != nil {
return
}
var delay time.Duration
if err != nil {
recovering = true
status := c.Status()
delay = c.retryDelay(status.Failures)
slog.Warn("bootstrap refresh failed; keeping last good snapshot",
"error", err, "failures", status.Failures, "retry_in", delay)
} else {
delay = c.refresh
switch {
case !wasLoaded:
slog.Info("bootstrap snapshot loaded", "sensors", c.Len())
case recovering:
slog.Info("bootstrap refresh recovered", "sensors", c.Len())
}
recovering = false
}
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
}
}
// Refresh performs one fetch. A decode/validation failure counts as a failed
// attempt and leaves the previous snapshot untouched.
func (c *Cache) Refresh(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()
payload, err := c.fetcher.Bootstrap(ctx)
now := c.now().UTC().Truncate(time.Second)
if err == nil {
var snapshot *snapshot
snapshot, err = newSnapshot(payload)
if err == nil {
c.mu.Lock()
c.snapshot = snapshot
c.loaded = true
c.status.LastSuccessAt = timePtr(now)
c.status.LastError = nil
c.status.NextRetryAt = nil
c.status.Failures = 0
c.mu.Unlock()
return nil
}
}
c.mu.Lock()
c.status.Failures++
message := err.Error()
c.status.LastError = &message
next := now.Add(c.retryDelay(c.status.Failures))
c.status.NextRetryAt = &next
c.mu.Unlock()
return err
}
func (c *Cache) retryDelay(failures int) time.Duration {
if failures < 1 {
failures = 1
}
if failures > len(c.retry) {
failures = len(c.retry)
}
return c.retry[failures-1]
}
func newSnapshot(payload model.Bootstrap) (*snapshot, error) {
switch {
case len(payload.Sensors) == 0:
return nil, fmt.Errorf("bootstrap snapshot: no sensors")
case payload.DataVersion == "":
return nil, fmt.Errorf("bootstrap snapshot: missing data_version")
case payload.ModelVersion == "":
return nil, fmt.Errorf("bootstrap snapshot: missing model_version")
case payload.ContextVersion == "":
return nil, fmt.Errorf("bootstrap snapshot: missing context_version")
case payload.Context.UpdatedAt.IsZero():
return nil, fmt.Errorf("bootstrap snapshot: missing context.updated_at")
}
snapshot := &snapshot{
DataVersion: payload.DataVersion,
ModelVersion: payload.ModelVersion,
ContextVersion: payload.ContextVersion,
Sensors: append([]model.Sensor(nil), payload.Sensors...),
Series: make(map[string][]model.Point, len(payload.Series)),
Context: payload.Context,
Coverage: payload.Meta.Coverage,
Metrics: payload.Meta.Metrics,
Limits: append([]string(nil), payload.Meta.Limits...),
sensorBy: make(map[string]model.Sensor, len(payload.Sensors)),
}
sort.Slice(snapshot.Sensors, func(i, j int) bool { return snapshot.Sensors[i].ID < snapshot.Sensors[j].ID })
for _, sensor := range snapshot.Sensors {
snapshot.sensorBy[sensor.ID] = sensor
}
for sensorID, points := range payload.Series {
copied := append([]model.Point(nil), points...)
sort.Slice(copied, func(i, j int) bool { return copied[i].T.Before(copied[j].T) })
snapshot.Series[sensorID] = copied
}
snapshot.Context.Holidays = append([]model.Holiday(nil), payload.Context.Holidays...)
snapshot.Context.Weather.Forecast = append([]model.WeatherPoint(nil), payload.Context.Weather.Forecast...)
return snapshot, nil
}
// Loaded reports whether a snapshot has ever been accepted.
func (c *Cache) Loaded() bool {
c.mu.RLock()
defer c.mu.RUnlock()
return c.loaded
}
// Ready reports whether a complete snapshot is available. FastAPI health is
// deliberately not part of this check: the cache itself is the safety net.
func (c *Cache) Ready() bool {
data, modelOK := c.Readiness()
return data && modelOK
}
// Readiness splits readiness into the data and model signals of /api/readyz.
func (c *Cache) Readiness() (data, modelOK bool) {
c.mu.RLock()
defer c.mu.RUnlock()
if !c.loaded || c.snapshot == nil {
return false, false
}
data = len(c.snapshot.Sensors) > 0 && !c.snapshot.Context.UpdatedAt.IsZero()
modelOK = c.snapshot.ModelVersion != ""
return data, modelOK
}
// Fresh reports whether the most recent refresh attempt succeeded. It is false
// before the first snapshot and after a failed refresh.
func (c *Cache) Fresh() bool {
c.mu.RLock()
defer c.mu.RUnlock()
return c.loaded && c.status.Failures == 0
}
// Status returns a copy of the cache status.
func (c *Cache) Status() model.CacheStatus {
c.mu.RLock()
defer c.mu.RUnlock()
return cloneStatus(c.status)
}
// Meta combines the snapshot metadata with the live cache status.
func (c *Cache) Meta() model.Meta {
c.mu.RLock()
defer c.mu.RUnlock()
meta := model.Meta{Cache: cloneStatus(c.status)}
if c.snapshot == nil {
return meta
}
meta.DataVersion = c.snapshot.DataVersion
meta.ModelVersion = c.snapshot.ModelVersion
meta.ContextVersion = c.snapshot.ContextVersion
meta.Coverage = c.snapshot.Coverage
meta.Metrics = c.snapshot.Metrics
meta.Limits = append([]string(nil), c.snapshot.Limits...)
return meta
}
// Sensors returns the sensors sorted by id.
func (c *Cache) Sensors() []model.Sensor {
c.mu.RLock()
snapshot := c.snapshot
c.mu.RUnlock()
if snapshot == nil {
return []model.Sensor{}
}
out := make([]model.Sensor, len(snapshot.Sensors))
copy(out, snapshot.Sensors)
return out
}
// Sensor looks up one sensor.
func (c *Cache) Sensor(id string) (model.Sensor, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return model.Sensor{}, false
}
sensor, ok := c.snapshot.sensorBy[id]
return sensor, ok
}
// Series returns the immutable series for a sensor, sorted by time.
func (c *Cache) Series(id string) ([]model.Point, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return nil, false
}
points, ok := c.snapshot.Series[id]
return points, ok
}
// Context returns the current weather/holiday context.
func (c *Cache) Context() model.ContextData {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return model.ContextData{}
}
return c.snapshot.Context
}
// Versions returns data, model and context version strings.
func (c *Cache) Versions() (data, modelVersion, contextVersion string) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return "", "", ""
}
return c.snapshot.DataVersion, c.snapshot.ModelVersion, c.snapshot.ContextVersion
}
// Len returns the number of sensors in the current snapshot.
func (c *Cache) Len() int {
c.mu.RLock()
defer c.mu.RUnlock()
if c.snapshot == nil {
return 0
}
return len(c.snapshot.Sensors)
}
func cloneStatus(s model.CacheStatus) model.CacheStatus {
out := s
if s.LastSuccessAt != nil {
v := *s.LastSuccessAt
out.LastSuccessAt = &v
}
if s.LastError != nil {
v := *s.LastError
out.LastError = &v
}
if s.NextRetryAt != nil {
v := *s.NextRetryAt
out.NextRetryAt = &v
}
return out
}
func timePtr(t time.Time) *time.Time { return &t }