Add scripts/integration.sh, which starts the real FastAPI and Go API, trains small and forecast through POST /api/train, checks SSE scenario propagation, heartbeat, reconnect, then kills FastAPI to verify snapshot and last-good forecast fallback before restarting it for recovery. Add scripts/sse-load, a standard-library load generator that holds 120 concurrent SSE streams for five minutes, posts scenario changes, records p50/p95/p99 event latency, drops and heartbeat cadence, and exits non-zero when a contract threshold is missed. Add unit tests, make targets and README documentation.
415 lines
20 KiB
Bash
Executable File
415 lines
20 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Repeatable end-to-end integration + fallback verification for the Go API.
|
|
#
|
|
# It starts the real FastAPI ML service on :18000 and the Go API on :18080,
|
|
# trains the small (and optionally forecast) model through POST /api/train,
|
|
# checks SSE scenario propagation / heartbeat / reconnect, kills FastAPI to
|
|
# prove the runtime snapshot + last-good forecast fallback, restarts it, and
|
|
# verifies recovery. With --load it also runs the 120-client 5-minute SSE test.
|
|
#
|
|
# scripts/integration.sh # fast checks (~1-2 min)
|
|
# scripts/integration.sh --load # plus the 5-minute load test
|
|
#
|
|
# Requirements: go, curl, jq, uv, and the ML checkout (read-only).
|
|
# The script owns both ports; it refuses to run if something is already there.
|
|
set -euo pipefail
|
|
|
|
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
|
ML_DIR=${ML_DIR:-/home/noel/Documents/Hackathon_2026/ML-py.git/main}
|
|
RAW_CSV=${RAW_CSV:-/home/noel/Documents/Hackathon_2026/data/raw/schwarz-pax.csv}
|
|
|
|
GO_PORT=${GO_PORT:-18080}
|
|
ML_PORT=${ML_PORT:-18000}
|
|
API_URL="http://127.0.0.1:${GO_PORT}"
|
|
ML_URL="http://127.0.0.1:${ML_PORT}"
|
|
|
|
CONTROL_USER=${CONTROL_USER:-control}
|
|
CONTROL_PASS=${CONTROL_PASS:-pulsecast-dev-pass}
|
|
CONTROL_TOKEN=${CONTROL_TOKEN:-pulsecast-control-token}
|
|
BOOTSTRAP_REFRESH=${BOOTSTRAP_REFRESH:-10s}
|
|
ML_TIMEOUT=${ML_TIMEOUT:-1500ms}
|
|
|
|
WORKDIR=${WORKDIR:-/tmp/pulsecast-integration}
|
|
RESULTS_DIR=${RESULTS_DIR:-$WORKDIR/results}
|
|
SMALL_CSV=${SMALL_CSV:-$WORKDIR/small.csv}
|
|
SMALL_FROM=${SMALL_FROM:-2026-09-01}
|
|
SMALL_TO=${SMALL_TO:-2026-10-16}
|
|
|
|
RUN_LOAD=0
|
|
RUN_FORECAST=${RUN_FORECAST:-1}
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--load) RUN_LOAD=1 ;;
|
|
--no-forecast) RUN_FORECAST=0 ;;
|
|
-h|--help) sed -n '2,14p' "$0"; exit 0 ;;
|
|
*) echo "unknown argument: $arg" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
mkdir -p "$WORKDIR" "$RESULTS_DIR"
|
|
API_LOG="$WORKDIR/api.log"
|
|
ML_LOG="$WORKDIR/ml.log"
|
|
API_PID=""
|
|
ML_PID=""
|
|
|
|
ms() { date +%s%3N; }
|
|
|
|
log() { printf '\n==> %s\n' "$*"; }
|
|
ok() { printf ' ok: %s\n' "$*"; }
|
|
fail() {
|
|
printf '\nFAIL: %s\n' "$*" >&2
|
|
printf 'api log: %s\nml log: %s\n' "$API_LOG" "$ML_LOG" >&2
|
|
exit 1
|
|
}
|
|
|
|
need() {
|
|
command -v "$1" >/dev/null 2>&1 || fail "missing command: $1"
|
|
}
|
|
|
|
port_busy() { ss -ltn 2>/dev/null | awk '{print $4}' | grep -q ":$1\$"; }
|
|
|
|
cleanup() {
|
|
local status=$?
|
|
if [[ -n "$API_PID" ]]; then
|
|
kill -TERM -- "-$API_PID" 2>/dev/null || kill -TERM "$API_PID" 2>/dev/null || true
|
|
fi
|
|
if [[ -n "$ML_PID" ]]; then
|
|
kill -TERM -- "-$ML_PID" 2>/dev/null || kill -TERM "$ML_PID" 2>/dev/null || true
|
|
fuser -k -TERM "$ML_PORT/tcp" >/dev/null 2>&1 || true
|
|
fi
|
|
exit "$status"
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
wait_http() { # url, deadline seconds
|
|
local url=$1 deadline=$2 start now
|
|
start=$(ms)
|
|
while :; do
|
|
if curl -sf -m 1 "$url" >/dev/null 2>&1; then return 0; fi
|
|
now=$(ms)
|
|
(( now - start < deadline * 1000 )) || return 1
|
|
sleep 0.2
|
|
done
|
|
}
|
|
|
|
json_get() { # file, jq filter
|
|
jq -r "$2" "$1" 2>/dev/null
|
|
}
|
|
|
|
get() { # path, outfile -> status code
|
|
curl -sS -m 10 -o "$2" -w '%{http_code}' "$API_URL$1"
|
|
}
|
|
|
|
need go
|
|
need curl
|
|
need jq
|
|
need uv
|
|
need awk
|
|
[[ -x "$ML_DIR/.venv/bin/python" || -f "$ML_DIR/pyproject.toml" ]] || fail "ML checkout not found at $ML_DIR"
|
|
[[ -f "$RAW_CSV" ]] || fail "raw CSV not found at $RAW_CSV"
|
|
port_busy "$GO_PORT" && fail "port $GO_PORT is already in use; stop that process first"
|
|
port_busy "$ML_PORT" && fail "port $ML_PORT is already in use; stop that process first"
|
|
|
|
# ------------------------------------------------------------------ small CSV
|
|
log "building a small training CSV ($SMALL_FROM .. $SMALL_TO)"
|
|
if [[ ! -s "$SMALL_CSV" || "$RAW_CSV" -nt "$SMALL_CSV" ]]; then
|
|
head -1 "$RAW_CSV" > "$SMALL_CSV.tmp"
|
|
awk -F',' -v from="\"$SMALL_FROM" -v to="\"$SMALL_TO" \
|
|
'NR>1 && $6 ~ /densityData/ && $1 >= from && $1 < to' "$RAW_CSV" >> "$SMALL_CSV.tmp"
|
|
mv "$SMALL_CSV.tmp" "$SMALL_CSV"
|
|
fi
|
|
small_rows=$(wc -l < "$SMALL_CSV")
|
|
(( small_rows > 1000 )) || fail "small CSV has only $small_rows lines"
|
|
ok "$small_rows rows, $(du -h "$SMALL_CSV" | cut -f1)"
|
|
|
|
# ------------------------------------------------------------------- FastAPI
|
|
log "starting FastAPI on :$ML_PORT"
|
|
(
|
|
cd "$ML_DIR"
|
|
env CONTROL_TOKEN="$CONTROL_TOKEN" \
|
|
PULSECAST_MODELS="$WORKDIR/models" \
|
|
PULSECAST_UPLOADS="$WORKDIR/uploads" \
|
|
setsid uv run --frozen uvicorn pulsecast.server:app \
|
|
--host 127.0.0.1 --port "$ML_PORT" > "$ML_LOG" 2>&1 &
|
|
echo $! > "$WORKDIR/ml.pid"
|
|
)
|
|
ML_PID=$(cat "$WORKDIR/ml.pid")
|
|
wait_http "$ML_URL/readyz" 90 || fail "FastAPI did not become ready; see $ML_LOG"
|
|
ok "readyz 200"
|
|
|
|
# ------------------------------------------------------------------- Go API
|
|
log "building and starting the Go API on :$GO_PORT"
|
|
( cd "$ROOT" && go build -o "$WORKDIR/api" ./cmd/api )
|
|
(
|
|
env PORT="$GO_PORT" FASTAPI_URL="$ML_URL" \
|
|
CONTROL_USER="$CONTROL_USER" CONTROL_PASS="$CONTROL_PASS" CONTROL_TOKEN="$CONTROL_TOKEN" \
|
|
BOOTSTRAP_REFRESH="$BOOTSTRAP_REFRESH" ML_TIMEOUT="$ML_TIMEOUT" \
|
|
setsid "$WORKDIR/api" > "$API_LOG" 2>&1 &
|
|
echo $! > "$WORKDIR/api.pid"
|
|
)
|
|
API_PID=$(cat "$WORKDIR/api.pid")
|
|
wait_http "$API_URL/api/readyz" 30 || fail "Go API did not become ready; see $API_LOG"
|
|
ok "readyz 200 after first /bootstrap snapshot"
|
|
|
|
initial_meta="$RESULTS_DIR/meta-initial.json"
|
|
get "/api/meta" "$initial_meta" >/dev/null
|
|
initial_success=$(json_get "$initial_meta" '.cache.last_success_at')
|
|
[[ -n "$initial_success" && "$initial_success" != "null" ]] || fail "no cache last_success_at"
|
|
ok "data_version=$(json_get "$initial_meta" '.data_version') model_version=$(json_get "$initial_meta" '.model_version')"
|
|
|
|
# ---------------------------------------------------------------- train small
|
|
poll_job() { # job_id, timeout seconds
|
|
local job=$1 timeout=$2 start now status
|
|
start=$(ms)
|
|
while :; do
|
|
curl -sS -m 10 -u "$CONTROL_USER:$CONTROL_PASS" \
|
|
"$API_URL/api/train/$job" > "$WORKDIR/job.json"
|
|
status=$(json_get "$WORKDIR/job.json" '.status')
|
|
if [[ "$status" == "succeeded" || "$status" == "failed" ]]; then return 0; fi
|
|
now=$(ms)
|
|
(( now - start < timeout * 1000 )) || return 1
|
|
sleep 0.05
|
|
done
|
|
}
|
|
|
|
run_train() { # model, timeout
|
|
local model=$1 timeout=$2 t0 submit_http status
|
|
t0=$(ms)
|
|
submit_http=$(curl -sS -m 60 -o "$WORKDIR/train-submit.json" -w '%{http_code}' \
|
|
-u "$CONTROL_USER:$CONTROL_PASS" \
|
|
-F "file=@$SMALL_CSV;type=text/csv" -F "model=$model" \
|
|
"$API_URL/api/train")
|
|
JOB_SUBMIT_MS=$(( $(ms) - t0 ))
|
|
JOB_SUBMIT_HTTP=$submit_http
|
|
[[ "$JOB_SUBMIT_HTTP" == "202" ]] || fail "POST /api/train ($model) returned $JOB_SUBMIT_HTTP: $(cat "$WORKDIR/train-submit.json")"
|
|
JOB_ID=$(json_get "$WORKDIR/train-submit.json" '.job_id')
|
|
[[ -n "$JOB_ID" && "$JOB_ID" != "null" ]] || fail "train response has no job_id"
|
|
poll_job "$JOB_ID" "$timeout" || fail "training job $JOB_ID ($model) did not finish within ${timeout}s"
|
|
JOB_TERMINAL_MS=$(( $(ms) - t0 ))
|
|
JOB_STATUS=$(json_get "$WORKDIR/job.json" '.status')
|
|
[[ "$JOB_STATUS" == "succeeded" ]] || fail "training job $JOB_ID ($model) status=$JOB_STATUS: $(cat "$WORKDIR/job.json")"
|
|
JOB_VERSION=$(json_get "$WORKDIR/job.json" '.model_version')
|
|
[[ -n "$JOB_VERSION" && "$JOB_VERSION" != "null" ]] || fail "job $JOB_ID has no model_version"
|
|
}
|
|
|
|
log "training model=small through POST /api/train (multipart CSV)"
|
|
run_train small 60
|
|
jq -e '.metrics.accuracy > 0.5 and .metrics.n_train > 0 and .metrics.n_test > 0 and .error == null' \
|
|
"$WORKDIR/job.json" >/dev/null || fail "small job metrics look wrong: $(cat "$WORKDIR/job.json")"
|
|
jq -n --argjson submit_http "$JOB_SUBMIT_HTTP" --argjson submit_ms "$JOB_SUBMIT_MS" \
|
|
--argjson terminal_ms "$JOB_TERMINAL_MS" --arg model small --slurpfile job "$WORKDIR/job.json" \
|
|
'{model:$model, submit_http:$submit_http, submit_ms:$submit_ms, submit_to_terminal_ms:$terminal_ms, job:$job[0]}' \
|
|
> "$RESULTS_DIR/train-small.json"
|
|
ok "job $JOB_ID succeeded in ${JOB_TERMINAL_MS}ms (submit ${JOB_SUBMIT_MS}ms); accuracy=$(json_get "$WORKDIR/job.json" '.metrics.accuracy') roc_auc=$(json_get "$WORKDIR/job.json" '.metrics.roc_auc') version=$JOB_VERSION"
|
|
|
|
forecast_before="$RESULTS_DIR/forecast-after-small.json"
|
|
code=$(get "/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base" "$forecast_before")
|
|
[[ "$code" == "200" ]] || fail "forecast after small training returned $code"
|
|
[[ "$(json_get "$forecast_before" '.cached')" == "false" ]] || fail "forecast after training is unexpectedly cached"
|
|
[[ "$(json_get "$forecast_before" '.points | length')" == "168" ]] || fail "forecast after training has wrong point count"
|
|
ok "live forecast still works: 168 points, cached=false"
|
|
|
|
if [[ "$RUN_FORECAST" == "1" ]]; then
|
|
log "training model=forecast through POST /api/train (multipart CSV)"
|
|
run_train forecast 300
|
|
jq -n --argjson submit_http "$JOB_SUBMIT_HTTP" --argjson submit_ms "$JOB_SUBMIT_MS" \
|
|
--argjson terminal_ms "$JOB_TERMINAL_MS" --arg model forecast --slurpfile job "$WORKDIR/job.json" \
|
|
'{model:$model, submit_http:$submit_http, submit_ms:$submit_ms, submit_to_terminal_ms:$terminal_ms, job:$job[0]}' \
|
|
> "$RESULTS_DIR/train-forecast.json"
|
|
ok "job $JOB_ID succeeded in ${JOB_TERMINAL_MS}ms; mae=$(json_get "$WORKDIR/job.json" '.metrics.mae') wape=$(json_get "$WORKDIR/job.json" '.metrics.wape') version=$JOB_VERSION"
|
|
|
|
# The snapshot refresh carries the hot-swapped model version to /api/meta.
|
|
deadline=$(($(ms) + 30000))
|
|
while :; do
|
|
get "/api/meta" "$WORKDIR/meta.json" >/dev/null
|
|
[[ "$(json_get "$WORKDIR/meta.json" '.model_version')" == "$JOB_VERSION" ]] && break
|
|
(( $(ms) < deadline )) || fail "Go cache did not pick up the retrained model_version"
|
|
sleep 0.5
|
|
done
|
|
ok "Go /api/meta model_version refreshed to $JOB_VERSION"
|
|
fi
|
|
|
|
# -------------------------------------------------------------- SSE scenario
|
|
log "checking SSE scenario propagation, heartbeat and reconnect"
|
|
sse_log="$RESULTS_DIR/sse-stream.log"
|
|
: > "$sse_log"
|
|
curl -sN "$API_URL/api/stream" > "$sse_log" &
|
|
SSE_PID=$!
|
|
sleep 0.5
|
|
grep -q '^retry: 3000$' "$sse_log" || fail "stream does not send retry: 3000"
|
|
grep -q '^event: scenario$' "$sse_log" || fail "stream does not send the current scenario on connect"
|
|
ok "connect: retry: 3000 + initial scenario event"
|
|
|
|
t0=$(ms)
|
|
curl -sS -u "$CONTROL_USER:$CONTROL_PASS" -H 'Content-Type: application/json' \
|
|
-d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0},"event":{"type":"market","uplift":0.2}}' \
|
|
"$API_URL/api/scenario" > "$WORKDIR/scenario.json"
|
|
posted_version=$(json_get "$WORKDIR/scenario.json" '.version')
|
|
[[ "${posted_version:-0}" -gt 1 ]] || fail "scenario POST did not return a new version: $(cat "$WORKDIR/scenario.json")"
|
|
propagation_ms=""
|
|
for _ in $(seq 1 200); do
|
|
if grep -q "^id: ${posted_version}$" "$sse_log"; then propagation_ms=$(( $(ms) - t0 )); break; fi
|
|
sleep 0.01
|
|
done
|
|
[[ -n "$propagation_ms" ]] || fail "scenario event did not reach the SSE client within 2s"
|
|
(( propagation_ms < 2000 )) || fail "scenario propagation took ${propagation_ms}ms (>2s)"
|
|
ok "scenario v$posted_version reached the client in ${propagation_ms}ms"
|
|
|
|
pings_before=$(grep -c '^event: ping$' "$sse_log" || true)
|
|
ping_t0=$(ms)
|
|
for _ in $(seq 1 180); do
|
|
if [[ "$(grep -c '^event: ping$' "$sse_log" || true)" -gt "$pings_before" ]]; then break; fi
|
|
sleep 0.1
|
|
done
|
|
[[ "$(grep -c '^event: ping$' "$sse_log" || true)" -gt "$pings_before" ]] || fail "no heartbeat within 18s"
|
|
heartbeat_ms=$(( $(ms) - ping_t0 ))
|
|
grep -q '^data: {"t":"' "$sse_log" || fail "heartbeat has no RFC3339 timestamp"
|
|
ok "heartbeat received ${heartbeat_ms}ms after the scenario check; server interval is 15s"
|
|
|
|
kill "$SSE_PID" 2>/dev/null || true
|
|
wait "$SSE_PID" 2>/dev/null || true
|
|
reconnect_t0=$(ms)
|
|
: > "$WORKDIR/sse-reconnect.log"
|
|
curl -sN "$API_URL/api/stream" > "$WORKDIR/sse-reconnect.log" &
|
|
SSE_PID=$!
|
|
reconnect_ms=""
|
|
for _ in $(seq 1 200); do
|
|
if grep -q "^id: ${posted_version}$" "$WORKDIR/sse-reconnect.log"; then reconnect_ms=$(( $(ms) - reconnect_t0 )); break; fi
|
|
sleep 0.01
|
|
done
|
|
kill "$SSE_PID" 2>/dev/null || true
|
|
wait "$SSE_PID" 2>/dev/null || true
|
|
[[ -n "$reconnect_ms" ]] || fail "reconnect did not receive the current scenario immediately"
|
|
ok "reconnect: current scenario v$posted_version delivered immediately"
|
|
|
|
jq -n --argjson version "$posted_version" --argjson propagation_ms "$propagation_ms" \
|
|
--argjson heartbeat_after_ms "$heartbeat_ms" --argjson reconnect_ms "$reconnect_ms" \
|
|
'{scenario_version:$version, propagation_ms:$propagation_ms, heartbeat_after_ms:$heartbeat_after_ms, heartbeat_interval_s:15, retry_ms:3000, reconnect_ms:$reconnect_ms, reconnect_current_version:$version}' \
|
|
> "$RESULTS_DIR/sse.json"
|
|
|
|
# ------------------------------------------------------- fallback + recovery
|
|
log "warming base forecast and staffing, then stopping FastAPI"
|
|
get "/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base" "$WORKDIR/warm-forecast.json" >/dev/null
|
|
[[ "$(json_get "$WORKDIR/warm-forecast.json" '.cached')" == "false" ]] || fail "warm forecast is not live"
|
|
get "/api/meta" "$WORKDIR/meta-warm.json" >/dev/null
|
|
last_success_before=$(json_get "$WORKDIR/meta-warm.json" '.cache.last_success_at')
|
|
|
|
kill -TERM -- "-$ML_PID" 2>/dev/null || kill -TERM "$ML_PID" 2>/dev/null || true
|
|
fuser -k -TERM "$ML_PORT/tcp" >/dev/null 2>&1 || true
|
|
for _ in $(seq 1 100); do curl -sf -m 1 "$ML_URL/healthz" >/dev/null 2>&1 || break; sleep 0.1; done
|
|
curl -sf -m 1 "$ML_URL/healthz" >/dev/null 2>&1 && fail "FastAPI is still reachable after kill"
|
|
ok "FastAPI stopped"
|
|
|
|
deadline=$(($(ms) + 45000))
|
|
while :; do
|
|
get "/api/meta" "$WORKDIR/meta-down.json" >/dev/null
|
|
failures=$(json_get "$WORKDIR/meta-down.json" '.cache.failures')
|
|
if [[ "${failures:-0}" -ge 1 ]]; then break; fi
|
|
(( $(ms) < deadline )) || fail "cache did not record a failed refresh within 45s"
|
|
sleep 0.25
|
|
done
|
|
jq -e '.cache.failures >= 1 and .cache.last_error != null and .cache.next_retry_at != null' \
|
|
"$WORKDIR/meta-down.json" >/dev/null || fail "cache status missing errors/next_retry_at"
|
|
last_success_down=$(json_get "$WORKDIR/meta-down.json" '.cache.last_success_at')
|
|
ok "cache failures=$failures last_error=$(json_get "$WORKDIR/meta-down.json" '.cache.last_error') next_retry_at=$(json_get "$WORKDIR/meta-down.json" '.cache.next_retry_at')"
|
|
|
|
code=$(curl -sS -o "$WORKDIR/readyz-down.json" -w '%{http_code}' "$API_URL/api/readyz")
|
|
[[ "$code" == "200" ]] || fail "readyz failed while FastAPI is down (status $code)"
|
|
ok "readyz still 200 (last good snapshot)"
|
|
|
|
code=$(get "/api/sensors" "$WORKDIR/sensors-down.json")
|
|
[[ "$code" == "200" ]] || fail "/api/sensors returned $code while FastAPI is down"
|
|
[[ "$(json_get "$WORKDIR/sensors-down.json" 'length')" == "8" ]] || fail "/api/sensors did not return 8 sensors"
|
|
code=$(get "/api/context" "$WORKDIR/context-down.json")
|
|
[[ "$code" == "200" ]] || fail "/api/context returned $code while FastAPI is down"
|
|
[[ "$(json_get "$WORKDIR/context-down.json" '.cached')" == "true" ]] || fail "/api/context is not marked cached"
|
|
code=$(get "/api/series?sensor=hn-stadtgalerie-01&step=1h" "$WORKDIR/series-down.json")
|
|
[[ "$code" == "200" ]] || fail "/api/series returned $code while FastAPI is down"
|
|
ok "sensors/context/series still served from the last good snapshot (cached=true)"
|
|
|
|
code=$(get "/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base" "$WORKDIR/forecast-down.json")
|
|
[[ "$code" == "200" ]] || fail "/api/forecast returned $code while FastAPI is down"
|
|
[[ "$(json_get "$WORKDIR/forecast-down.json" '.cached')" == "true" ]] || fail "fallback forecast is not marked cached"
|
|
[[ "$(json_get "$WORKDIR/forecast-down.json" '.points | length')" == "168" ]] || fail "fallback forecast has wrong point count"
|
|
ok "forecast serves the last good 168 points with cached=true"
|
|
|
|
code=$(get "/api/staffing?sensor=hn-stadtgalerie-01&scenario=base&standard=standard" "$WORKDIR/staffing-down.json")
|
|
[[ "$code" == "200" ]] || fail "/api/staffing returned $code while FastAPI is down"
|
|
[[ "$(json_get "$WORKDIR/staffing-down.json" '.cached')" == "true" ]] || fail "fallback staffing is not marked cached"
|
|
ok "staffing builds from the last good forecast with cached=true"
|
|
|
|
code=$(get "/api/forecast?sensor=density-01&horizon=15m&scenario=rainy-saturday" "$WORKDIR/forecast-cold.json")
|
|
[[ "$code" == "503" ]] || fail "uncached forecast returned $code, want 503"
|
|
ok "uncached forecast correctly returns 503 (no invented data)"
|
|
|
|
fallback_failures=$failures
|
|
fallback_error=$(json_get "$WORKDIR/meta-down.json" '.cache.last_error')
|
|
fallback_next=$(json_get "$WORKDIR/meta-down.json" '.cache.next_retry_at')
|
|
|
|
log "restarting FastAPI and waiting for recovery"
|
|
(
|
|
cd "$ML_DIR"
|
|
env CONTROL_TOKEN="$CONTROL_TOKEN" \
|
|
PULSECAST_MODELS="$WORKDIR/models" \
|
|
PULSECAST_UPLOADS="$WORKDIR/uploads" \
|
|
setsid uv run --frozen uvicorn pulsecast.server:app \
|
|
--host 127.0.0.1 --port "$ML_PORT" > "$ML_LOG" 2>&1 &
|
|
echo $! > "$WORKDIR/ml.pid"
|
|
)
|
|
ML_PID=$(cat "$WORKDIR/ml.pid")
|
|
wait_http "$ML_URL/readyz" 90 || fail "FastAPI did not restart; see $ML_LOG"
|
|
recovery_start=$(ms)
|
|
deadline=$(($(ms) + 120000))
|
|
while :; do
|
|
get "/api/meta" "$WORKDIR/meta-recovered.json" >/dev/null
|
|
failures=$(json_get "$WORKDIR/meta-recovered.json" '.cache.failures')
|
|
if [[ "${failures:-1}" == "0" ]]; then break; fi
|
|
(( $(ms) < deadline )) || fail "cache did not recover within 120s"
|
|
sleep 0.25
|
|
done
|
|
recovery_ms=$(( $(ms) - recovery_start ))
|
|
recovered_success=$(json_get "$WORKDIR/meta-recovered.json" '.cache.last_success_at')
|
|
[[ "$recovered_success" != "$last_success_down" ]] || fail "last_success_at did not advance after recovery"
|
|
code=$(get "/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base" "$WORKDIR/forecast-recovered.json")
|
|
[[ "$code" == "200" ]] || fail "forecast after recovery returned $code"
|
|
[[ "$(json_get "$WORKDIR/forecast-recovered.json" '.cached')" == "false" ]] || fail "forecast after recovery is still cached"
|
|
ok "recovered after ${recovery_ms}ms: failures=0, last_success_at=$recovered_success"
|
|
|
|
jq -n \
|
|
--argjson fallback_failures "$fallback_failures" \
|
|
--arg fallback_error "$fallback_error" \
|
|
--arg fallback_next_retry "$fallback_next" \
|
|
--argjson fallback_forecast_cached "$(json_get "$WORKDIR/forecast-down.json" '.cached')" \
|
|
--argjson fallback_staffing_cached "$(json_get "$WORKDIR/staffing-down.json" '.cached')" \
|
|
--argjson recovery_ms "$recovery_ms" \
|
|
--arg last_success_before "$last_success_before" \
|
|
--arg last_success_down "$last_success_down" \
|
|
--arg last_success_after "$recovered_success" \
|
|
--argjson recovered_cached "$(json_get "$WORKDIR/forecast-recovered.json" '.cached')" \
|
|
'{fastapi_down:{cache_failures:$fallback_failures, last_error:$fallback_error, next_retry_at:$fallback_next_retry, last_success_at:$last_success_down, forecast_cached:$fallback_forecast_cached, staffing_cached:$fallback_staffing_cached, readyz:200, sensors:8}, recovery:{after_ms:$recovery_ms, last_success_before:$last_success_before, last_success_after:$last_success_after, forecast_cached:$recovered_cached, failures:0}}' \
|
|
> "$RESULTS_DIR/fallback.json"
|
|
|
|
# ------------------------------------------------------------------ load test
|
|
if [[ "$RUN_LOAD" == "1" ]]; then
|
|
log "running the 120-client SSE load test (5 minutes)"
|
|
( cd "$ROOT" && CONTROL_USER="$CONTROL_USER" CONTROL_PASS="$CONTROL_PASS" \
|
|
go run ./scripts/sse-load -base "$API_URL" -clients 120 -duration 5m \
|
|
-changes 30s -json "$RESULTS_DIR/load.json" ) | tee "$RESULTS_DIR/load.txt"
|
|
jq -e '.passed == true' "$RESULTS_DIR/load.json" >/dev/null || fail "SSE load test failed; see $RESULTS_DIR/load.json"
|
|
ok "load test passed"
|
|
fi
|
|
|
|
log "all integration checks passed"
|
|
cat > "$RESULTS_DIR/summary.txt" <<EOF
|
|
PulseCast Go integration $(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
FastAPI $ML_URL, Go $API_URL
|
|
small CSV: $small_rows rows ($SMALL_FROM .. $SMALL_TO)
|
|
small job: $(jq -r '"status=\(.job.status) version=\(.job.model_version) accuracy=\(.job.metrics.accuracy) roc_auc=\(.job.metrics.roc_auc) submit_to_terminal_ms=\(.submit_to_terminal_ms)"' "$RESULTS_DIR/train-small.json")
|
|
SSE: $(jq -c '.' "$RESULTS_DIR/sse.json")
|
|
fallback: $(jq -c '.' "$RESULTS_DIR/fallback.json")
|
|
EOF
|
|
if [[ "$RUN_FORECAST" == "1" ]]; then
|
|
echo "forecast job: $(jq -r '"status=\(.job.status) version=\(.job.model_version) mae=\(.job.metrics.mae) wape=\(.job.metrics.wape) submit_to_terminal_ms=\(.submit_to_terminal_ms)"' "$RESULTS_DIR/train-forecast.json")" >> "$RESULTS_DIR/summary.txt"
|
|
fi
|
|
cat "$RESULTS_DIR/summary.txt"
|