feat: runtime bootstrap snapshot cache with backoff
This commit is contained in:
1 parent
08f60a14d1
commit
aa6bb23e24
19 files changed
+1400
-384
No files matched your search
@@ -20,7 +20,11 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
|
||||
return
|
||||
}
|
||||
if _, ok := s.store.Sensor(sensor); !ok {
|
||||
if s.cache == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
if _, ok := s.cache.Sensor(sensor); !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
@@ -53,11 +57,14 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
slog.Warn("forecast unavailable", "sensor", sensor, "horizon", horizon, "scenario", scenarioName, "error", err)
|
||||
writeError(w, http.StatusServiceUnavailable, "forecast_unavailable",
|
||||
"no live forecast and no baked forecast for this sensor, scenario and horizon")
|
||||
"no live forecast and no last good forecast for this sensor, scenario and horizon")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
s.rememberForecast(sensor, horizon, scenarioName, points)
|
||||
}
|
||||
|
||||
meta := s.meta()
|
||||
s.setVersions(w)
|
||||
writeJSON(w, http.StatusOK, model.ForecastResponse{
|
||||
Sensor: sensor,
|
||||
@@ -65,9 +72,9 @@ func (s *Server) handleForecast(w http.ResponseWriter, r *http.Request) {
|
||||
Scenario: scenarioName,
|
||||
Points: points,
|
||||
Cached: cached,
|
||||
DataVersion: s.store.Meta().DataVersion,
|
||||
ModelVersion: s.store.Meta().ModelVersion,
|
||||
ContextVersion: s.store.Meta().ContextVersion,
|
||||
DataVersion: meta.DataVersion,
|
||||
ModelVersion: meta.ModelVersion,
|
||||
ContextVersion: meta.ContextVersion,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -78,7 +85,7 @@ func (s *Server) liveForecast(ctx context.Context, sensor, horizon, scenarioName
|
||||
ctx, cancel := context.WithTimeout(ctx, s.cfg.MLTimeout)
|
||||
defer cancel()
|
||||
|
||||
ctxData := s.store.Context()
|
||||
ctxData := s.cache.Context()
|
||||
precip := ctxData.Weather.Now.PrecipMM
|
||||
if state.Scenario.Weather.PrecipMM > 0 {
|
||||
precip = state.Scenario.Weather.PrecipMM
|
||||
@@ -100,8 +107,9 @@ func (s *Server) liveForecast(ctx context.Context, sensor, horizon, scenarioName
|
||||
return sortedForecast(points), nil
|
||||
}
|
||||
|
||||
// fallbackForecast prefers live /simulate output for the current scenario, then
|
||||
// the baked scenario grid. It never labels base data as another scenario.
|
||||
// fallbackForecast prefers the live /simulate grid for the current scenario,
|
||||
// then the last good /predict response for this exact key. It never labels base
|
||||
// data as another scenario.
|
||||
func (s *Server) fallbackForecast(sensor, scenarioName, horizon string) ([]model.ForecastPoint, bool) {
|
||||
s.mu.RLock()
|
||||
if s.simFor == scenarioName {
|
||||
@@ -115,10 +123,31 @@ func (s *Server) fallbackForecast(sensor, scenarioName, horizon string) ([]model
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
if points, ok := s.store.Forecast(sensor, scenarioName, horizon); ok {
|
||||
return sortedForecast(points), true
|
||||
return s.lastGoodForecast(sensor, horizon, scenarioName)
|
||||
}
|
||||
|
||||
func forecastKey(sensor, horizon, scenarioName string) string {
|
||||
return sensor + "|" + horizon + "|" + scenarioName
|
||||
}
|
||||
|
||||
func (s *Server) rememberForecast(sensor, horizon, scenarioName string, points []model.ForecastPoint) {
|
||||
if len(points) == 0 {
|
||||
return
|
||||
}
|
||||
return nil, false
|
||||
copied := sortedForecast(points)
|
||||
s.mu.Lock()
|
||||
s.lastForecast[forecastKey(sensor, horizon, scenarioName)] = copied
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *Server) lastGoodForecast(sensor, horizon, scenarioName string) ([]model.ForecastPoint, bool) {
|
||||
s.mu.RLock()
|
||||
points, ok := s.lastForecast[forecastKey(sensor, horizon, scenarioName)]
|
||||
s.mu.RUnlock()
|
||||
if !ok || len(points) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
return sortedForecast(points), true
|
||||
}
|
||||
|
||||
func sortedForecast(points []model.ForecastPoint) []model.ForecastPoint {
|
||||
@@ -129,8 +158,11 @@ func sortedForecast(points []model.ForecastPoint) []model.ForecastPoint {
|
||||
}
|
||||
|
||||
func (s *Server) holidayOn(t time.Time) (string, bool) {
|
||||
if s.cache == nil {
|
||||
return "", false
|
||||
}
|
||||
day := t.In(s.loc).Format("2006-01-02")
|
||||
for _, holiday := range s.store.Context().Holidays {
|
||||
for _, holiday := range s.cache.Context().Holidays {
|
||||
if holiday.Date == day {
|
||||
return holiday.Name, true
|
||||
}
|
||||
@@ -145,7 +177,11 @@ func (s *Server) handleStaffing(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "sensor is required")
|
||||
return
|
||||
}
|
||||
if _, ok := s.store.Sensor(sensor); !ok {
|
||||
if s.cache == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
if _, ok := s.cache.Sensor(sensor); !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "unknown sensor")
|
||||
return
|
||||
}
|
||||
@@ -169,45 +205,49 @@ func (s *Server) handleStaffing(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
state := s.scenario.Get()
|
||||
points, cached, ok := s.staffingForecast(sensor, scenarioName)
|
||||
if !ok {
|
||||
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable",
|
||||
"no 7d forecast for this sensor and scenario")
|
||||
return
|
||||
}
|
||||
meta := s.meta()
|
||||
|
||||
s.mu.RLock()
|
||||
generation := s.simGen
|
||||
s.mu.RUnlock()
|
||||
key := fmt.Sprintf("%s|%s|%s|%d|%d", sensor, scenarioName, standard, state.Version, generation)
|
||||
key := fmt.Sprintf("%s|%s|%s|%s|%d|%d", sensor, scenarioName, standard,
|
||||
meta.ContextVersion, state.Version, generation)
|
||||
|
||||
s.mu.RLock()
|
||||
plan, hit := s.staffing[key]
|
||||
s.mu.RUnlock()
|
||||
|
||||
if !hit {
|
||||
built, err := staffing.Build(sensor, scenarioName, standard, points, s.store.Context())
|
||||
points, cached, ok := s.staffingForecast(r.Context(), sensor, scenarioName, state)
|
||||
if !ok {
|
||||
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable",
|
||||
"no live or last good 7d forecast for this sensor and scenario")
|
||||
return
|
||||
}
|
||||
built, err := staffing.Build(sensor, scenarioName, standard, points, s.cache.Context())
|
||||
if err != nil {
|
||||
slog.Warn("staffing build failed", "sensor", sensor, "scenario", scenarioName, "standard", standard, "error", err)
|
||||
writeError(w, http.StatusServiceUnavailable, "staffing_unavailable", "could not build a plan from the available forecast")
|
||||
return
|
||||
}
|
||||
built.Cached = cached
|
||||
s.mu.Lock()
|
||||
s.staffing[key] = built
|
||||
s.mu.Unlock()
|
||||
plan = built
|
||||
}
|
||||
|
||||
plan.Cached = cached
|
||||
plan.DataVersion = s.store.Meta().DataVersion
|
||||
plan.ModelVersion = s.store.Meta().ModelVersion
|
||||
plan.ContextVersion = s.store.Meta().ContextVersion
|
||||
plan.DataVersion = meta.DataVersion
|
||||
plan.ModelVersion = meta.ModelVersion
|
||||
plan.ContextVersion = meta.ContextVersion
|
||||
s.setVersions(w)
|
||||
writeJSON(w, http.StatusOK, plan)
|
||||
}
|
||||
|
||||
// staffingForecast returns the best 7d points and whether they came from the
|
||||
// fallback grid rather than live /simulate output.
|
||||
func (s *Server) staffingForecast(sensor, scenarioName string) ([]model.ForecastPoint, bool, bool) {
|
||||
// staffingForecast returns 7d points for a plan. Prefer the scenario simulation
|
||||
// grid, then /predict, and finally the last good forecast. The bool reports
|
||||
// whether the points came from the cache fallback.
|
||||
func (s *Server) staffingForecast(ctx context.Context, sensor, scenarioName string, state model.ScenarioState) ([]model.ForecastPoint, bool, bool) {
|
||||
s.mu.RLock()
|
||||
if s.simFor == scenarioName {
|
||||
if horizons, ok := s.sim[sensor]; ok {
|
||||
@@ -220,8 +260,14 @@ func (s *Server) staffingForecast(sensor, scenarioName string) ([]model.Forecast
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
if points, ok := s.store.Forecast(sensor, scenarioName, "7d"); ok {
|
||||
return sortedForecast(points), scenarioName != "base", true
|
||||
points, err := s.liveForecast(ctx, sensor, "7d", scenarioName, state)
|
||||
if err == nil {
|
||||
s.rememberForecast(sensor, "7d", scenarioName, points)
|
||||
return points, false, true
|
||||
}
|
||||
slog.Warn("staffing forecast fallback", "sensor", sensor, "scenario", scenarioName, "error", err)
|
||||
if points, ok := s.lastGoodForecast(sensor, "7d", scenarioName); ok {
|
||||
return points, true, true
|
||||
}
|
||||
return nil, false, false
|
||||
}
|
||||
Reference in new issue
Block a user