feat: runtime bootstrap snapshot cache with backoff
This commit is contained in:
1 parent
08f60a14d1
commit
aa6bb23e24
19 files changed
+1400
-384
No files matched your search
Vendored
+358
@@ -0,0 +1,358 @@
|
||||
// Package cache keeps the FastAPI /bootstrap snapshot in memory and refreshes
|
||||
// it in the background. A failed refresh never discards the last good snapshot;
|
||||
// the next attempt follows the contract backoff 1/2/4/8/16/30 s (cap).
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pulsecast/api/internal/model"
|
||||
)
|
||||
|
||||
// Source is the only snapshot producer: the FastAPI /bootstrap endpoint.
|
||||
const Source = "fastapi"
|
||||
|
||||
// DefaultRefreshInterval is how often a healthy snapshot is refreshed.
|
||||
const DefaultRefreshInterval = 60 * time.Second
|
||||
|
||||
// DefaultTimeout bounds one /bootstrap request.
|
||||
const DefaultTimeout = 20 * time.Second
|
||||
|
||||
// DefaultRetryDelays is the contract backoff schedule. The last value repeats.
|
||||
var DefaultRetryDelays = []time.Duration{
|
||||
1 * time.Second,
|
||||
2 * time.Second,
|
||||
4 * time.Second,
|
||||
8 * time.Second,
|
||||
16 * time.Second,
|
||||
30 * time.Second,
|
||||
}
|
||||
|
||||
// Fetcher is the FastAPI bootstrap surface. *ml.Client implements it.
|
||||
type Fetcher interface {
|
||||
Bootstrap(ctx context.Context) (model.Bootstrap, error)
|
||||
}
|
||||
|
||||
// Options configures refresh behaviour. Zero values use the defaults.
|
||||
type Options struct {
|
||||
RefreshInterval time.Duration
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// snapshot is one immutable /bootstrap payload with its lookup indexes.
|
||||
type snapshot struct {
|
||||
DataVersion string
|
||||
ModelVersion string
|
||||
ContextVersion string
|
||||
Sensors []model.Sensor
|
||||
Series map[string][]model.Point
|
||||
Context model.ContextData
|
||||
Coverage model.Coverage
|
||||
Metrics model.Metrics
|
||||
Limits []string
|
||||
|
||||
sensorBy map[string]model.Sensor
|
||||
}
|
||||
|
||||
// Cache holds the current snapshot and its refresh status. All methods are
|
||||
// safe for concurrent use.
|
||||
type Cache struct {
|
||||
fetcher Fetcher
|
||||
refresh time.Duration
|
||||
timeout time.Duration
|
||||
retry []time.Duration
|
||||
now func() time.Time
|
||||
mu sync.RWMutex
|
||||
snapshot *snapshot
|
||||
loaded bool
|
||||
status model.CacheStatus
|
||||
}
|
||||
|
||||
// New returns an empty cache. Call Run (or Refresh) to load the first snapshot.
|
||||
func New(fetcher Fetcher, opts Options) *Cache {
|
||||
if opts.RefreshInterval <= 0 {
|
||||
opts.RefreshInterval = DefaultRefreshInterval
|
||||
}
|
||||
if opts.Timeout <= 0 {
|
||||
opts.Timeout = DefaultTimeout
|
||||
}
|
||||
return &Cache{
|
||||
fetcher: fetcher,
|
||||
refresh: opts.RefreshInterval,
|
||||
timeout: opts.Timeout,
|
||||
retry: append([]time.Duration(nil), DefaultRetryDelays...),
|
||||
now: time.Now,
|
||||
status: model.CacheStatus{Source: Source},
|
||||
}
|
||||
}
|
||||
|
||||
// Run loads the snapshot immediately, then refreshes it. Failures keep the last
|
||||
// good snapshot and retry with the contract backoff. Run returns when ctx ends.
|
||||
func (c *Cache) Run(ctx context.Context) {
|
||||
recovering := false
|
||||
for {
|
||||
wasLoaded := c.Loaded()
|
||||
err := c.Refresh(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var delay time.Duration
|
||||
if err != nil {
|
||||
recovering = true
|
||||
status := c.Status()
|
||||
delay = c.retryDelay(status.Failures)
|
||||
slog.Warn("bootstrap refresh failed; keeping last good snapshot",
|
||||
"error", err, "failures", status.Failures, "retry_in", delay)
|
||||
} else {
|
||||
delay = c.refresh
|
||||
switch {
|
||||
case !wasLoaded:
|
||||
slog.Info("bootstrap snapshot loaded", "sensors", c.Len())
|
||||
case recovering:
|
||||
slog.Info("bootstrap refresh recovered", "sensors", c.Len())
|
||||
}
|
||||
recovering = false
|
||||
}
|
||||
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh performs one fetch. A decode/validation failure counts as a failed
|
||||
// attempt and leaves the previous snapshot untouched.
|
||||
func (c *Cache) Refresh(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
||||
defer cancel()
|
||||
|
||||
payload, err := c.fetcher.Bootstrap(ctx)
|
||||
now := c.now().UTC().Truncate(time.Second)
|
||||
if err == nil {
|
||||
var snapshot *snapshot
|
||||
snapshot, err = newSnapshot(payload)
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
c.snapshot = snapshot
|
||||
c.loaded = true
|
||||
c.status.LastSuccessAt = timePtr(now)
|
||||
c.status.LastError = nil
|
||||
c.status.NextRetryAt = nil
|
||||
c.status.Failures = 0
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
c.status.Failures++
|
||||
message := err.Error()
|
||||
c.status.LastError = &message
|
||||
next := now.Add(c.retryDelay(c.status.Failures))
|
||||
c.status.NextRetryAt = &next
|
||||
c.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cache) retryDelay(failures int) time.Duration {
|
||||
if failures < 1 {
|
||||
failures = 1
|
||||
}
|
||||
if failures > len(c.retry) {
|
||||
failures = len(c.retry)
|
||||
}
|
||||
return c.retry[failures-1]
|
||||
}
|
||||
|
||||
func newSnapshot(payload model.Bootstrap) (*snapshot, error) {
|
||||
switch {
|
||||
case len(payload.Sensors) == 0:
|
||||
return nil, fmt.Errorf("bootstrap snapshot: no sensors")
|
||||
case payload.DataVersion == "":
|
||||
return nil, fmt.Errorf("bootstrap snapshot: missing data_version")
|
||||
case payload.ModelVersion == "":
|
||||
return nil, fmt.Errorf("bootstrap snapshot: missing model_version")
|
||||
case payload.ContextVersion == "":
|
||||
return nil, fmt.Errorf("bootstrap snapshot: missing context_version")
|
||||
case payload.Context.UpdatedAt.IsZero():
|
||||
return nil, fmt.Errorf("bootstrap snapshot: missing context.updated_at")
|
||||
}
|
||||
|
||||
snapshot := &snapshot{
|
||||
DataVersion: payload.DataVersion,
|
||||
ModelVersion: payload.ModelVersion,
|
||||
ContextVersion: payload.ContextVersion,
|
||||
Sensors: append([]model.Sensor(nil), payload.Sensors...),
|
||||
Series: make(map[string][]model.Point, len(payload.Series)),
|
||||
Context: payload.Context,
|
||||
Coverage: payload.Meta.Coverage,
|
||||
Metrics: payload.Meta.Metrics,
|
||||
Limits: append([]string(nil), payload.Meta.Limits...),
|
||||
sensorBy: make(map[string]model.Sensor, len(payload.Sensors)),
|
||||
}
|
||||
sort.Slice(snapshot.Sensors, func(i, j int) bool { return snapshot.Sensors[i].ID < snapshot.Sensors[j].ID })
|
||||
for _, sensor := range snapshot.Sensors {
|
||||
snapshot.sensorBy[sensor.ID] = sensor
|
||||
}
|
||||
for sensorID, points := range payload.Series {
|
||||
copied := append([]model.Point(nil), points...)
|
||||
sort.Slice(copied, func(i, j int) bool { return copied[i].T.Before(copied[j].T) })
|
||||
snapshot.Series[sensorID] = copied
|
||||
}
|
||||
snapshot.Context.Holidays = append([]model.Holiday(nil), payload.Context.Holidays...)
|
||||
snapshot.Context.Weather.Forecast = append([]model.WeatherPoint(nil), payload.Context.Weather.Forecast...)
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// Loaded reports whether a snapshot has ever been accepted.
|
||||
func (c *Cache) Loaded() bool {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.loaded
|
||||
}
|
||||
|
||||
// Ready reports whether a complete snapshot is available. FastAPI health is
|
||||
// deliberately not part of this check: the cache itself is the safety net.
|
||||
func (c *Cache) Ready() bool {
|
||||
data, modelOK := c.Readiness()
|
||||
return data && modelOK
|
||||
}
|
||||
|
||||
// Readiness splits readiness into the data and model signals of /api/readyz.
|
||||
func (c *Cache) Readiness() (data, modelOK bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if !c.loaded || c.snapshot == nil {
|
||||
return false, false
|
||||
}
|
||||
data = len(c.snapshot.Sensors) > 0 && !c.snapshot.Context.UpdatedAt.IsZero()
|
||||
modelOK = c.snapshot.ModelVersion != ""
|
||||
return data, modelOK
|
||||
}
|
||||
|
||||
// Fresh reports whether the most recent refresh attempt succeeded. It is false
|
||||
// before the first snapshot and after a failed refresh.
|
||||
func (c *Cache) Fresh() bool {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return c.loaded && c.status.Failures == 0
|
||||
}
|
||||
|
||||
// Status returns a copy of the cache status.
|
||||
func (c *Cache) Status() model.CacheStatus {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return cloneStatus(c.status)
|
||||
}
|
||||
|
||||
// Meta combines the snapshot metadata with the live cache status.
|
||||
func (c *Cache) Meta() model.Meta {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
meta := model.Meta{Cache: cloneStatus(c.status)}
|
||||
if c.snapshot == nil {
|
||||
return meta
|
||||
}
|
||||
meta.DataVersion = c.snapshot.DataVersion
|
||||
meta.ModelVersion = c.snapshot.ModelVersion
|
||||
meta.ContextVersion = c.snapshot.ContextVersion
|
||||
meta.Coverage = c.snapshot.Coverage
|
||||
meta.Metrics = c.snapshot.Metrics
|
||||
meta.Limits = append([]string(nil), c.snapshot.Limits...)
|
||||
return meta
|
||||
}
|
||||
|
||||
// Sensors returns the sensors sorted by id.
|
||||
func (c *Cache) Sensors() []model.Sensor {
|
||||
c.mu.RLock()
|
||||
snapshot := c.snapshot
|
||||
c.mu.RUnlock()
|
||||
if snapshot == nil {
|
||||
return []model.Sensor{}
|
||||
}
|
||||
out := make([]model.Sensor, len(snapshot.Sensors))
|
||||
copy(out, snapshot.Sensors)
|
||||
return out
|
||||
}
|
||||
|
||||
// Sensor looks up one sensor.
|
||||
func (c *Cache) Sensor(id string) (model.Sensor, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return model.Sensor{}, false
|
||||
}
|
||||
sensor, ok := c.snapshot.sensorBy[id]
|
||||
return sensor, ok
|
||||
}
|
||||
|
||||
// Series returns the immutable series for a sensor, sorted by time.
|
||||
func (c *Cache) Series(id string) ([]model.Point, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return nil, false
|
||||
}
|
||||
points, ok := c.snapshot.Series[id]
|
||||
return points, ok
|
||||
}
|
||||
|
||||
// Context returns the current weather/holiday context.
|
||||
func (c *Cache) Context() model.ContextData {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return model.ContextData{}
|
||||
}
|
||||
return c.snapshot.Context
|
||||
}
|
||||
|
||||
// Versions returns data, model and context version strings.
|
||||
func (c *Cache) Versions() (data, modelVersion, contextVersion string) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return "", "", ""
|
||||
}
|
||||
return c.snapshot.DataVersion, c.snapshot.ModelVersion, c.snapshot.ContextVersion
|
||||
}
|
||||
|
||||
// Len returns the number of sensors in the current snapshot.
|
||||
func (c *Cache) Len() int {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
if c.snapshot == nil {
|
||||
return 0
|
||||
}
|
||||
return len(c.snapshot.Sensors)
|
||||
}
|
||||
|
||||
func cloneStatus(s model.CacheStatus) model.CacheStatus {
|
||||
out := s
|
||||
if s.LastSuccessAt != nil {
|
||||
v := *s.LastSuccessAt
|
||||
out.LastSuccessAt = &v
|
||||
}
|
||||
if s.LastError != nil {
|
||||
v := *s.LastError
|
||||
out.LastError = &v
|
||||
}
|
||||
if s.NextRetryAt != nil {
|
||||
v := *s.NextRetryAt
|
||||
out.NextRetryAt = &v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func timePtr(t time.Time) *time.Time { return &t }
|
||||
Reference in new issue
Block a user