feat: runtime bootstrap snapshot cache with backoff

This commit is contained in:
Noel Sabia committed 2026-11-14 06:08:53 +01:00
1 parent 08f60a14d1
commit aa6bb23e24
19 files changed
+1400 -384

No files matched your search

+76 -30
View File
@@ -20,7 +20,11 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
return
}
if _, ok := s.store.Sensor(sensor); !ok {
if s.cache == nil {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
if _, ok := s.cache.Sensor(sensor); !ok {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
@@ -53,11 +57,14 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
if !ok {
slog.Warn("forecast unavailable", "sensor", sensor, "horizon", horizon, "scenario", scenarioName, "error", err)
writeError(w, http.StatusServiceUnavailable, "forecast_unavailable",
"no live forecast and no baked forecast for this sensor, scenario and horizon")
"no live forecast and no last good forecast for this sensor, scenario and horizon")
return
}
} else {
s.rememberForecast(sensor, horizon, scenarioName, points)
}
meta := s.meta()
s.setVersions(w)
writeJSON(w, http.StatusOK, model.ForecastResponse{
Sensor: sensor,
@@ -65,9 +72,9 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
Scenario: scenarioName,
Points: points,
Cached: cached,
DataVersion: s.store.Meta().DataVersion,
ModelVersion: s.store.Meta().ModelVersion,
ContextVersion: s.store.Meta().ContextVersion,
DataVersion: meta.DataVersion,
ModelVersion: meta.ModelVersion,
ContextVersion: meta.ContextVersion,
})
}
@@ -78,7 +85,7 @@ func (s *Server) liveForecast(ctx context.Context, sensor, horizon, scenarioName
ctx, cancel := context.WithTimeout(ctx, s.cfg.MLTimeout)
defer cancel()
ctxData := s.store.Context()
ctxData := s.cache.Context()
precip := ctxData.Weather.Now.PrecipMM
if state.Scenario.Weather.PrecipMM > 0 {
precip = state.Scenario.Weather.PrecipMM
@@ -100,8 +107,9 @@ func (s *Server) liveForecast(ctx context.Context, sensor, horizon, scenarioName
return sortedForecast(points), nil
}
// fallbackForecast prefers live /simulate output for the current scenario, then
// the baked scenario grid. It never labels base data as another scenario.
// fallbackForecast prefers the live /simulate grid for the current scenario,
// then the last good /predict response for this exact key. It never labels base
// data as another scenario.
func (s *Server) fallbackForecast(sensor, scenarioName, horizon string) ([]model.ForecastPoint, bool) {
s.mu.RLock()
if s.simFor == scenarioName {
@@ -115,10 +123,31 @@ func (s *Server) fallbackForecast(sensor, scenarioName, horizon string) ([]model
}
s.mu.RUnlock()
if points, ok := s.store.Forecast(sensor, scenarioName, horizon); ok {
return sortedForecast(points), true
return s.lastGoodForecast(sensor, horizon, scenarioName)
}
func forecastKey(sensor, horizon, scenarioName string) string {
return sensor + "|" + horizon + "|" + scenarioName
}
func (s *Server) rememberForecast(sensor, horizon, scenarioName string, points []model.ForecastPoint) {
if len(points) == 0 {
return
}
return nil, false
copied := sortedForecast(points)
s.mu.Lock()
s.lastForecast[forecastKey(sensor, horizon, scenarioName)] = copied
s.mu.Unlock()
}
func (s *Server) lastGoodForecast(sensor, horizon, scenarioName string) ([]model.ForecastPoint, bool) {
s.mu.RLock()
points, ok := s.lastForecast[forecastKey(sensor, horizon, scenarioName)]
s.mu.RUnlock()
if !ok || len(points) == 0 {
return nil, false
}
return sortedForecast(points), true
}
func sortedForecast(points []model.ForecastPoint) []model.ForecastPoint {
@@ -129,8 +158,11 @@ func sortedForecast(points []model.ForecastPoint) []model.ForecastPoint {
}
func (s *Server) holidayOn(t time.Time) (string, bool) {
if s.cache == nil {
return "", false
}
day := t.In(s.loc).Format("2006-01-02")
for _, holiday := range s.store.Context().Holidays {
for _, holiday := range s.cache.Context().Holidays {
if holiday.Date == day {
return holiday.Name, true
}
@@ -145,7 +177,11 @@ func (s *Server) handleStaffing(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
return
}
if _, ok := s.store.Sensor(sensor); !ok {
if s.cache == nil {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
if _, ok := s.cache.Sensor(sensor); !ok {
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
return
}
@@ -169,45 +205,49 @@ func (s *Server) handleStaffing(w http.ResponseWriter, r *http.Request) {
}
state := s.scenario.Get()
points, cached, ok := s.staffingForecast(sensor, scenarioName)
if !ok {
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable",
"no 7d forecast for this sensor and scenario")
return
}
meta := s.meta()
s.mu.RLock()
generation := s.simGen
s.mu.RUnlock()
key := fmt.Sprintf("%s|%s|%s|%d|%d", sensor, scenarioName, standard, state.Version, generation)
key := fmt.Sprintf("%s|%s|%s|%s|%d|%d", sensor, scenarioName, standard,
meta.ContextVersion, state.Version, generation)
s.mu.RLock()
plan, hit := s.staffing[key]
s.mu.RUnlock()
if !hit {
built, err := staffing.Build(sensor, scenarioName, standard, points, s.store.Context())
points, cached, ok := s.staffingForecast(r.Context(), sensor, scenarioName, state)
if !ok {
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable",
"no live or last good 7d forecast for this sensor and scenario")
return
}
built, err := staffing.Build(sensor, scenarioName, standard, points, s.cache.Context())
if err != nil {
slog.Warn("staffing build failed", "sensor", sensor, "scenario", scenarioName, "standard", standard, "error", err)
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable", "could not build a plan from the available forecast")
return
}
built.Cached = cached
s.mu.Lock()
s.staffing[key] = built
s.mu.Unlock()
plan = built
}
plan.Cached = cached
plan.DataVersion = s.store.Meta().DataVersion
plan.ModelVersion = s.store.Meta().ModelVersion
plan.ContextVersion = s.store.Meta().ContextVersion
plan.DataVersion = meta.DataVersion
plan.ModelVersion = meta.ModelVersion
plan.ContextVersion = meta.ContextVersion
s.setVersions(w)
writeJSON(w, http.StatusOK, plan)
}
// staffingForecast returns the best 7d points and whether they came from the
// fallback grid rather than live /simulate output.
func (s *Server) staffingForecast(sensor, scenarioName string) ([]model.ForecastPoint, bool, bool) {
// staffingForecast returns 7d points for a plan. Prefer the scenario simulation
// grid, then /predict, and finally the last good forecast. The bool reports
// whether the points came from the cache fallback.
func (s *Server) staffingForecast(ctx context.Context, sensor, scenarioName string, state model.ScenarioState) ([]model.ForecastPoint, bool, bool) {
s.mu.RLock()
if s.simFor == scenarioName {
if horizons, ok := s.sim[sensor]; ok {
@@ -220,8 +260,14 @@ func (s *Server) staffingForecast(sensor, scenarioName string) ([]model.Forecast
}
s.mu.RUnlock()
if points, ok := s.store.Forecast(sensor, scenarioName, "7d"); ok {
return sortedForecast(points), scenarioName != "base", true
points, err := s.liveForecast(ctx, sensor, "7d", scenarioName, state)
if err == nil {
s.rememberForecast(sensor, "7d", scenarioName, points)
return points, false, true
}
slog.Warn("staffing forecast fallback", "sensor", sensor, "scenario", scenarioName, "error", err)
if points, ok := s.lastGoodForecast(sensor, "7d", scenarioName); ok {
return points, true, true
}
return nil, false, false
}