diff --git a/Makefile b/Makefile index f52692b..cccf701 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GO ?= go -.PHONY: check vet test build +.PHONY: check vet test build integration integration-load load # Local equivalent of the Gitea Actions CI job. check: vet test build @@ -13,3 +13,16 @@ test: build: $(GO) build ./... + +# End-to-end: real FastAPI on :18000, Go on :18080, train small+forecast, +# SSE scenario/heartbeat/reconnect, FastAPI kill + cache fallback + recovery. +integration: + scripts/integration.sh + +# The same run plus the 120-client, 5-minute SSE load test. +integration-load: + scripts/integration.sh --load + +# Load-only against an already-running API (CONTROL_USER/CONTROL_PASS from env). +load: + $(GO) run ./scripts/sse-load -base http://127.0.0.1:18080 -clients 120 -duration 5m diff --git a/README.md b/README.md index 807521f..9b9fece 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,33 @@ the last good snapshot and retries with exponential backoff | `BOOTSTRAP_REFRESH` | `60s` | Healthy snapshot refresh interval | | `BOOTSTRAP_TIMEOUT` | `20s` | Per-request `/bootstrap` timeout | | `ML_TIMEOUT` | `1500ms` | Per-request `/predict` timeout | +| `CONTROL_USER` / `CONTROL_PASS` | unset | Basic auth for `POST /api/scenario` and `/api/train*` | +| `CONTROL_TOKEN` | unset | Optional Bearer token forwarded to FastAPI `/train*` | + +## Integration and load test + +`make integration` runs the real stack (it refuses to start if `:18000` or +`:18080` is busy): + +1. starts FastAPI from the ML checkout on `:18000` and the Go API on `:18080`; +2. uploads a 24k-row CSV subset to `POST /api/train` and polls the job + (`small`, then `forecast` by default); +3. verifies the live forecast still answers, scenario propagation, the 15 s + heartbeat and reconnect behaviour; +4. stops FastAPI, verifies `/api/meta.cache`, snapshot-served sensors, the + last-good forecast (`cached: true`) and staffing fallback; +5. restarts FastAPI and waits for the cache to recover. + +`make integration-load` additionally runs `scripts/sse-load`: 120 concurrent +SSE clients for 5 minutes with a scenario change every 30 s. It records peak +connections, drops, per-change and overall p95 latency, and heartbeat cadence. +The tool exits non-zero when a client drops, an event is missing, or p95 +scenario latency exceeds the contract's 2 s threshold. + +Results land in `/tmp/pulsecast-integration/results` by default. Override with +`WORKDIR=/path` or `RESULTS_DIR=/path`. `make load` runs only the load test +against an API that is already up. + ## Container diff --git a/scripts/integration.sh b/scripts/integration.sh new file mode 100755 index 0000000..fccf137 --- /dev/null +++ b/scripts/integration.sh @@ -0,0 +1,414 @@ +#!/usr/bin/env bash +# Repeatable end-to-end integration + fallback verification for the Go API. +# +# It starts the real FastAPI ML service on :18000 and the Go API on :18080, +# trains the small (and optionally forecast) model through POST /api/train, +# checks SSE scenario propagation / heartbeat / reconnect, kills FastAPI to +# prove the runtime snapshot + last-good forecast fallback, restarts it, and +# verifies recovery. With --load it also runs the 120-client 5-minute SSE test. +# +# scripts/integration.sh # fast checks (~1-2 min) +# scripts/integration.sh --load # plus the 5-minute load test +# +# Requirements: go, curl, jq, uv, and the ML checkout (read-only). +# The script owns both ports; it refuses to run if something is already there. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +ML_DIR=${ML_DIR:-/home/noel/Documents/Hackathon_2026/ML-py.git/main} +RAW_CSV=${RAW_CSV:-/home/noel/Documents/Hackathon_2026/data/raw/schwarz-pax.csv} + +GO_PORT=${GO_PORT:-18080} +ML_PORT=${ML_PORT:-18000} +API_URL="http://127.0.0.1:${GO_PORT}" +ML_URL="http://127.0.0.1:${ML_PORT}" + +CONTROL_USER=${CONTROL_USER:-control} +CONTROL_PASS=${CONTROL_PASS:-pulsecast-dev-pass} +CONTROL_TOKEN=${CONTROL_TOKEN:-pulsecast-control-token} +BOOTSTRAP_REFRESH=${BOOTSTRAP_REFRESH:-10s} +ML_TIMEOUT=${ML_TIMEOUT:-1500ms} + +WORKDIR=${WORKDIR:-/tmp/pulsecast-integration} +RESULTS_DIR=${RESULTS_DIR:-$WORKDIR/results} +SMALL_CSV=${SMALL_CSV:-$WORKDIR/small.csv} +SMALL_FROM=${SMALL_FROM:-2026-09-01} +SMALL_TO=${SMALL_TO:-2026-10-16} + +RUN_LOAD=0 +RUN_FORECAST=${RUN_FORECAST:-1} +for arg in "$@"; do + case "$arg" in + --load) RUN_LOAD=1 ;; + --no-forecast) RUN_FORECAST=0 ;; + -h|--help) sed -n '2,14p' "$0"; exit 0 ;; + *) echo "unknown argument: $arg" >&2; exit 2 ;; + esac +done + +mkdir -p "$WORKDIR" "$RESULTS_DIR" +API_LOG="$WORKDIR/api.log" +ML_LOG="$WORKDIR/ml.log" +API_PID="" +ML_PID="" + +ms() { date +%s%3N; } + +log() { printf '\n==> %s\n' "$*"; } +ok() { printf ' ok: %s\n' "$*"; } +fail() { + printf '\nFAIL: %s\n' "$*" >&2 + printf 'api log: %s\nml log: %s\n' "$API_LOG" "$ML_LOG" >&2 + exit 1 +} + +need() { + command -v "$1" >/dev/null 2>&1 || fail "missing command: $1" +} + +port_busy() { ss -ltn 2>/dev/null | awk '{print $4}' | grep -q ":$1\$"; } + +cleanup() { + local status=$? + if [[ -n "$API_PID" ]]; then + kill -TERM -- "-$API_PID" 2>/dev/null || kill -TERM "$API_PID" 2>/dev/null || true + fi + if [[ -n "$ML_PID" ]]; then + kill -TERM -- "-$ML_PID" 2>/dev/null || kill -TERM "$ML_PID" 2>/dev/null || true + fuser -k -TERM "$ML_PORT/tcp" >/dev/null 2>&1 || true + fi + exit "$status" +} +trap cleanup EXIT INT TERM + +wait_http() { # url, deadline seconds + local url=$1 deadline=$2 start now + start=$(ms) + while :; do + if curl -sf -m 1 "$url" >/dev/null 2>&1; then return 0; fi + now=$(ms) + (( now - start < deadline * 1000 )) || return 1 + sleep 0.2 + done +} + +json_get() { # file, jq filter + jq -r "$2" "$1" 2>/dev/null +} + +get() { # path, outfile -> status code + curl -sS -m 10 -o "$2" -w '%{http_code}' "$API_URL$1" +} + +need go +need curl +need jq +need uv +need awk +[[ -x "$ML_DIR/.venv/bin/python" || -f "$ML_DIR/pyproject.toml" ]] || fail "ML checkout not found at $ML_DIR" +[[ -f "$RAW_CSV" ]] || fail "raw CSV not found at $RAW_CSV" +port_busy "$GO_PORT" && fail "port $GO_PORT is already in use; stop that process first" +port_busy "$ML_PORT" && fail "port $ML_PORT is already in use; stop that process first" + +# ------------------------------------------------------------------ small CSV +log "building a small training CSV ($SMALL_FROM .. $SMALL_TO)" +if [[ ! -s "$SMALL_CSV" || "$RAW_CSV" -nt "$SMALL_CSV" ]]; then + head -1 "$RAW_CSV" > "$SMALL_CSV.tmp" + awk -F',' -v from="\"$SMALL_FROM" -v to="\"$SMALL_TO" \ + 'NR>1 && $6 ~ /densityData/ && $1 >= from && $1 < to' "$RAW_CSV" >> "$SMALL_CSV.tmp" + mv "$SMALL_CSV.tmp" "$SMALL_CSV" +fi +small_rows=$(wc -l < "$SMALL_CSV") +(( small_rows > 1000 )) || fail "small CSV has only $small_rows lines" +ok "$small_rows rows, $(du -h "$SMALL_CSV" | cut -f1)" + +# ------------------------------------------------------------------- FastAPI +log "starting FastAPI on :$ML_PORT" +( + cd "$ML_DIR" + env CONTROL_TOKEN="$CONTROL_TOKEN" \ + PULSECAST_MODELS="$WORKDIR/models" \ + PULSECAST_UPLOADS="$WORKDIR/uploads" \ + setsid uv run --frozen uvicorn pulsecast.server:app \ + --host 127.0.0.1 --port "$ML_PORT" > "$ML_LOG" 2>&1 & + echo $! > "$WORKDIR/ml.pid" +) +ML_PID=$(cat "$WORKDIR/ml.pid") +wait_http "$ML_URL/readyz" 90 || fail "FastAPI did not become ready; see $ML_LOG" +ok "readyz 200" + +# ------------------------------------------------------------------- Go API +log "building and starting the Go API on :$GO_PORT" +( cd "$ROOT" && go build -o "$WORKDIR/api" ./cmd/api ) +( + env PORT="$GO_PORT" FASTAPI_URL="$ML_URL" \ + CONTROL_USER="$CONTROL_USER" CONTROL_PASS="$CONTROL_PASS" CONTROL_TOKEN="$CONTROL_TOKEN" \ + BOOTSTRAP_REFRESH="$BOOTSTRAP_REFRESH" ML_TIMEOUT="$ML_TIMEOUT" \ + setsid "$WORKDIR/api" > "$API_LOG" 2>&1 & + echo $! > "$WORKDIR/api.pid" +) +API_PID=$(cat "$WORKDIR/api.pid") +wait_http "$API_URL/api/readyz" 30 || fail "Go API did not become ready; see $API_LOG" +ok "readyz 200 after first /bootstrap snapshot" + +initial_meta="$RESULTS_DIR/meta-initial.json" +get "/api/meta" "$initial_meta" >/dev/null +initial_success=$(json_get "$initial_meta" '.cache.last_success_at') +[[ -n "$initial_success" && "$initial_success" != "null" ]] || fail "no cache last_success_at" +ok "data_version=$(json_get "$initial_meta" '.data_version') model_version=$(json_get "$initial_meta" '.model_version')" + +# ---------------------------------------------------------------- train small +poll_job() { # job_id, timeout seconds + local job=$1 timeout=$2 start now status + start=$(ms) + while :; do + curl -sS -m 10 -u "$CONTROL_USER:$CONTROL_PASS" \ + "$API_URL/api/train/$job" > "$WORKDIR/job.json" + status=$(json_get "$WORKDIR/job.json" '.status') + if [[ "$status" == "succeeded" || "$status" == "failed" ]]; then return 0; fi + now=$(ms) + (( now - start < timeout * 1000 )) || return 1 + sleep 0.05 + done +} + +run_train() { # model, timeout + local model=$1 timeout=$2 t0 submit_http status + t0=$(ms) + submit_http=$(curl -sS -m 60 -o "$WORKDIR/train-submit.json" -w '%{http_code}' \ + -u "$CONTROL_USER:$CONTROL_PASS" \ + -F "file=@$SMALL_CSV;type=text/csv" -F "model=$model" \ + "$API_URL/api/train") + JOB_SUBMIT_MS=$(( $(ms) - t0 )) + JOB_SUBMIT_HTTP=$submit_http + [[ "$JOB_SUBMIT_HTTP" == "202" ]] || fail "POST /api/train ($model) returned $JOB_SUBMIT_HTTP: $(cat "$WORKDIR/train-submit.json")" + JOB_ID=$(json_get "$WORKDIR/train-submit.json" '.job_id') + [[ -n "$JOB_ID" && "$JOB_ID" != "null" ]] || fail "train response has no job_id" + poll_job "$JOB_ID" "$timeout" || fail "training job $JOB_ID ($model) did not finish within ${timeout}s" + JOB_TERMINAL_MS=$(( $(ms) - t0 )) + JOB_STATUS=$(json_get "$WORKDIR/job.json" '.status') + [[ "$JOB_STATUS" == "succeeded" ]] || fail "training job $JOB_ID ($model) status=$JOB_STATUS: $(cat "$WORKDIR/job.json")" + JOB_VERSION=$(json_get "$WORKDIR/job.json" '.model_version') + [[ -n "$JOB_VERSION" && "$JOB_VERSION" != "null" ]] || fail "job $JOB_ID has no model_version" +} + +log "training model=small through POST /api/train (multipart CSV)" +run_train small 60 +jq -e '.metrics.accuracy > 0.5 and .metrics.n_train > 0 and .metrics.n_test > 0 and .error == null' \ + "$WORKDIR/job.json" >/dev/null || fail "small job metrics look wrong: $(cat "$WORKDIR/job.json")" +jq -n --argjson submit_http "$JOB_SUBMIT_HTTP" --argjson submit_ms "$JOB_SUBMIT_MS" \ + --argjson terminal_ms "$JOB_TERMINAL_MS" --arg model small --slurpfile job "$WORKDIR/job.json" \ + '{model:$model, submit_http:$submit_http, submit_ms:$submit_ms, submit_to_terminal_ms:$terminal_ms, job:$job[0]}' \ + > "$RESULTS_DIR/train-small.json" +ok "job $JOB_ID succeeded in ${JOB_TERMINAL_MS}ms (submit ${JOB_SUBMIT_MS}ms); accuracy=$(json_get "$WORKDIR/job.json" '.metrics.accuracy') roc_auc=$(json_get "$WORKDIR/job.json" '.metrics.roc_auc') version=$JOB_VERSION" + +forecast_before="$RESULTS_DIR/forecast-after-small.json" +code=$(get "/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base" "$forecast_before") +[[ "$code" == "200" ]] || fail "forecast after small training returned $code" +[[ "$(json_get "$forecast_before" '.cached')" == "false" ]] || fail "forecast after training is unexpectedly cached" +[[ "$(json_get "$forecast_before" '.points | length')" == "168" ]] || fail "forecast after training has wrong point count" +ok "live forecast still works: 168 points, cached=false" + +if [[ "$RUN_FORECAST" == "1" ]]; then + log "training model=forecast through POST /api/train (multipart CSV)" + run_train forecast 300 + jq -n --argjson submit_http "$JOB_SUBMIT_HTTP" --argjson submit_ms "$JOB_SUBMIT_MS" \ + --argjson terminal_ms "$JOB_TERMINAL_MS" --arg model forecast --slurpfile job "$WORKDIR/job.json" \ + '{model:$model, submit_http:$submit_http, submit_ms:$submit_ms, submit_to_terminal_ms:$terminal_ms, job:$job[0]}' \ + > "$RESULTS_DIR/train-forecast.json" + ok "job $JOB_ID succeeded in ${JOB_TERMINAL_MS}ms; mae=$(json_get "$WORKDIR/job.json" '.metrics.mae') wape=$(json_get "$WORKDIR/job.json" '.metrics.wape') version=$JOB_VERSION" + + # The snapshot refresh carries the hot-swapped model version to /api/meta. + deadline=$(($(ms) + 30000)) + while :; do + get "/api/meta" "$WORKDIR/meta.json" >/dev/null + [[ "$(json_get "$WORKDIR/meta.json" '.model_version')" == "$JOB_VERSION" ]] && break + (( $(ms) < deadline )) || fail "Go cache did not pick up the retrained model_version" + sleep 0.5 + done + ok "Go /api/meta model_version refreshed to $JOB_VERSION" +fi + +# -------------------------------------------------------------- SSE scenario +log "checking SSE scenario propagation, heartbeat and reconnect" +sse_log="$RESULTS_DIR/sse-stream.log" +: > "$sse_log" +curl -sN "$API_URL/api/stream" > "$sse_log" & +SSE_PID=$! +sleep 0.5 +grep -q '^retry: 3000$' "$sse_log" || fail "stream does not send retry: 3000" +grep -q '^event: scenario$' "$sse_log" || fail "stream does not send the current scenario on connect" +ok "connect: retry: 3000 + initial scenario event" + +t0=$(ms) +curl -sS -u "$CONTROL_USER:$CONTROL_PASS" -H 'Content-Type: application/json' \ + -d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0},"event":{"type":"market","uplift":0.2}}' \ + "$API_URL/api/scenario" > "$WORKDIR/scenario.json" +posted_version=$(json_get "$WORKDIR/scenario.json" '.version') +[[ "${posted_version:-0}" -gt 1 ]] || fail "scenario POST did not return a new version: $(cat "$WORKDIR/scenario.json")" +propagation_ms="" +for _ in $(seq 1 200); do + if grep -q "^id: ${posted_version}$" "$sse_log"; then propagation_ms=$(( $(ms) - t0 )); break; fi + sleep 0.01 +done +[[ -n "$propagation_ms" ]] || fail "scenario event did not reach the SSE client within 2s" +(( propagation_ms < 2000 )) || fail "scenario propagation took ${propagation_ms}ms (>2s)" +ok "scenario v$posted_version reached the client in ${propagation_ms}ms" + +pings_before=$(grep -c '^event: ping$' "$sse_log" || true) +ping_t0=$(ms) +for _ in $(seq 1 180); do + if [[ "$(grep -c '^event: ping$' "$sse_log" || true)" -gt "$pings_before" ]]; then break; fi + sleep 0.1 +done +[[ "$(grep -c '^event: ping$' "$sse_log" || true)" -gt "$pings_before" ]] || fail "no heartbeat within 18s" +heartbeat_ms=$(( $(ms) - ping_t0 )) +grep -q '^data: {"t":"' "$sse_log" || fail "heartbeat has no RFC3339 timestamp" +ok "heartbeat received ${heartbeat_ms}ms after the scenario check; server interval is 15s" + +kill "$SSE_PID" 2>/dev/null || true +wait "$SSE_PID" 2>/dev/null || true +reconnect_t0=$(ms) +: > "$WORKDIR/sse-reconnect.log" +curl -sN "$API_URL/api/stream" > "$WORKDIR/sse-reconnect.log" & +SSE_PID=$! +reconnect_ms="" +for _ in $(seq 1 200); do + if grep -q "^id: ${posted_version}$" "$WORKDIR/sse-reconnect.log"; then reconnect_ms=$(( $(ms) - reconnect_t0 )); break; fi + sleep 0.01 +done +kill "$SSE_PID" 2>/dev/null || true +wait "$SSE_PID" 2>/dev/null || true +[[ -n "$reconnect_ms" ]] || fail "reconnect did not receive the current scenario immediately" +ok "reconnect: current scenario v$posted_version delivered immediately" + +jq -n --argjson version "$posted_version" --argjson propagation_ms "$propagation_ms" \ + --argjson heartbeat_after_ms "$heartbeat_ms" --argjson reconnect_ms "$reconnect_ms" \ + '{scenario_version:$version, propagation_ms:$propagation_ms, heartbeat_after_ms:$heartbeat_after_ms, heartbeat_interval_s:15, retry_ms:3000, reconnect_ms:$reconnect_ms, reconnect_current_version:$version}' \ + > "$RESULTS_DIR/sse.json" + +# ------------------------------------------------------- fallback + recovery +log "warming base forecast and staffing, then stopping FastAPI" +get "/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base" "$WORKDIR/warm-forecast.json" >/dev/null +[[ "$(json_get "$WORKDIR/warm-forecast.json" '.cached')" == "false" ]] || fail "warm forecast is not live" +get "/api/meta" "$WORKDIR/meta-warm.json" >/dev/null +last_success_before=$(json_get "$WORKDIR/meta-warm.json" '.cache.last_success_at') + +kill -TERM -- "-$ML_PID" 2>/dev/null || kill -TERM "$ML_PID" 2>/dev/null || true +fuser -k -TERM "$ML_PORT/tcp" >/dev/null 2>&1 || true +for _ in $(seq 1 100); do curl -sf -m 1 "$ML_URL/healthz" >/dev/null 2>&1 || break; sleep 0.1; done +curl -sf -m 1 "$ML_URL/healthz" >/dev/null 2>&1 && fail "FastAPI is still reachable after kill" +ok "FastAPI stopped" + +deadline=$(($(ms) + 45000)) +while :; do + get "/api/meta" "$WORKDIR/meta-down.json" >/dev/null + failures=$(json_get "$WORKDIR/meta-down.json" '.cache.failures') + if [[ "${failures:-0}" -ge 1 ]]; then break; fi + (( $(ms) < deadline )) || fail "cache did not record a failed refresh within 45s" + sleep 0.25 +done +jq -e '.cache.failures >= 1 and .cache.last_error != null and .cache.next_retry_at != null' \ + "$WORKDIR/meta-down.json" >/dev/null || fail "cache status missing errors/next_retry_at" +last_success_down=$(json_get "$WORKDIR/meta-down.json" '.cache.last_success_at') +ok "cache failures=$failures last_error=$(json_get "$WORKDIR/meta-down.json" '.cache.last_error') next_retry_at=$(json_get "$WORKDIR/meta-down.json" '.cache.next_retry_at')" + +code=$(curl -sS -o "$WORKDIR/readyz-down.json" -w '%{http_code}' "$API_URL/api/readyz") +[[ "$code" == "200" ]] || fail "readyz failed while FastAPI is down (status $code)" +ok "readyz still 200 (last good snapshot)" + +code=$(get "/api/sensors" "$WORKDIR/sensors-down.json") +[[ "$code" == "200" ]] || fail "/api/sensors returned $code while FastAPI is down" +[[ "$(json_get "$WORKDIR/sensors-down.json" 'length')" == "8" ]] || fail "/api/sensors did not return 8 sensors" +code=$(get "/api/context" "$WORKDIR/context-down.json") +[[ "$code" == "200" ]] || fail "/api/context returned $code while FastAPI is down" +[[ "$(json_get "$WORKDIR/context-down.json" '.cached')" == "true" ]] || fail "/api/context is not marked cached" +code=$(get "/api/series?sensor=hn-stadtgalerie-01&step=1h" "$WORKDIR/series-down.json") +[[ "$code" == "200" ]] || fail "/api/series returned $code while FastAPI is down" +ok "sensors/context/series still served from the last good snapshot (cached=true)" + +code=$(get "/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base" "$WORKDIR/forecast-down.json") +[[ "$code" == "200" ]] || fail "/api/forecast returned $code while FastAPI is down" +[[ "$(json_get "$WORKDIR/forecast-down.json" '.cached')" == "true" ]] || fail "fallback forecast is not marked cached" +[[ "$(json_get "$WORKDIR/forecast-down.json" '.points | length')" == "168" ]] || fail "fallback forecast has wrong point count" +ok "forecast serves the last good 168 points with cached=true" + +code=$(get "/api/staffing?sensor=hn-stadtgalerie-01&scenario=base&standard=standard" "$WORKDIR/staffing-down.json") +[[ "$code" == "200" ]] || fail "/api/staffing returned $code while FastAPI is down" +[[ "$(json_get "$WORKDIR/staffing-down.json" '.cached')" == "true" ]] || fail "fallback staffing is not marked cached" +ok "staffing builds from the last good forecast with cached=true" + +code=$(get "/api/forecast?sensor=density-01&horizon=15m&scenario=rainy-saturday" "$WORKDIR/forecast-cold.json") +[[ "$code" == "503" ]] || fail "uncached forecast returned $code, want 503" +ok "uncached forecast correctly returns 503 (no invented data)" + +fallback_failures=$failures +fallback_error=$(json_get "$WORKDIR/meta-down.json" '.cache.last_error') +fallback_next=$(json_get "$WORKDIR/meta-down.json" '.cache.next_retry_at') + +log "restarting FastAPI and waiting for recovery" +( + cd "$ML_DIR" + env CONTROL_TOKEN="$CONTROL_TOKEN" \ + PULSECAST_MODELS="$WORKDIR/models" \ + PULSECAST_UPLOADS="$WORKDIR/uploads" \ + setsid uv run --frozen uvicorn pulsecast.server:app \ + --host 127.0.0.1 --port "$ML_PORT" > "$ML_LOG" 2>&1 & + echo $! > "$WORKDIR/ml.pid" +) +ML_PID=$(cat "$WORKDIR/ml.pid") +wait_http "$ML_URL/readyz" 90 || fail "FastAPI did not restart; see $ML_LOG" +recovery_start=$(ms) +deadline=$(($(ms) + 120000)) +while :; do + get "/api/meta" "$WORKDIR/meta-recovered.json" >/dev/null + failures=$(json_get "$WORKDIR/meta-recovered.json" '.cache.failures') + if [[ "${failures:-1}" == "0" ]]; then break; fi + (( $(ms) < deadline )) || fail "cache did not recover within 120s" + sleep 0.25 +done +recovery_ms=$(( $(ms) - recovery_start )) +recovered_success=$(json_get "$WORKDIR/meta-recovered.json" '.cache.last_success_at') +[[ "$recovered_success" != "$last_success_down" ]] || fail "last_success_at did not advance after recovery" +code=$(get "/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base" "$WORKDIR/forecast-recovered.json") +[[ "$code" == "200" ]] || fail "forecast after recovery returned $code" +[[ "$(json_get "$WORKDIR/forecast-recovered.json" '.cached')" == "false" ]] || fail "forecast after recovery is still cached" +ok "recovered after ${recovery_ms}ms: failures=0, last_success_at=$recovered_success" + +jq -n \ + --argjson fallback_failures "$fallback_failures" \ + --arg fallback_error "$fallback_error" \ + --arg fallback_next_retry "$fallback_next" \ + --argjson fallback_forecast_cached "$(json_get "$WORKDIR/forecast-down.json" '.cached')" \ + --argjson fallback_staffing_cached "$(json_get "$WORKDIR/staffing-down.json" '.cached')" \ + --argjson recovery_ms "$recovery_ms" \ + --arg last_success_before "$last_success_before" \ + --arg last_success_down "$last_success_down" \ + --arg last_success_after "$recovered_success" \ + --argjson recovered_cached "$(json_get "$WORKDIR/forecast-recovered.json" '.cached')" \ + '{fastapi_down:{cache_failures:$fallback_failures, last_error:$fallback_error, next_retry_at:$fallback_next_retry, last_success_at:$last_success_down, forecast_cached:$fallback_forecast_cached, staffing_cached:$fallback_staffing_cached, readyz:200, sensors:8}, recovery:{after_ms:$recovery_ms, last_success_before:$last_success_before, last_success_after:$last_success_after, forecast_cached:$recovered_cached, failures:0}}' \ + > "$RESULTS_DIR/fallback.json" + +# ------------------------------------------------------------------ load test +if [[ "$RUN_LOAD" == "1" ]]; then + log "running the 120-client SSE load test (5 minutes)" + ( cd "$ROOT" && CONTROL_USER="$CONTROL_USER" CONTROL_PASS="$CONTROL_PASS" \ + go run ./scripts/sse-load -base "$API_URL" -clients 120 -duration 5m \ + -changes 30s -json "$RESULTS_DIR/load.json" ) | tee "$RESULTS_DIR/load.txt" + jq -e '.passed == true' "$RESULTS_DIR/load.json" >/dev/null || fail "SSE load test failed; see $RESULTS_DIR/load.json" + ok "load test passed" +fi + +log "all integration checks passed" +cat > "$RESULTS_DIR/summary.txt" <> "$RESULTS_DIR/summary.txt" +fi +cat "$RESULTS_DIR/summary.txt" diff --git a/scripts/sse-load/main.go b/scripts/sse-load/main.go new file mode 100755 index 0000000..90e3480 --- /dev/null +++ b/scripts/sse-load/main.go @@ -0,0 +1,694 @@ +// Command sse-load is the acceptance load test for GET /api/stream. +// +// It opens N concurrent SSE clients (120 by default) for a fixed duration, +// posts scenario changes on an interval, and measures the end-to-end latency +// from the POST to each client receiving the matching scenario event. It also +// verifies heartbeat cadence, counts connection drops/reconnects, and writes a +// machine-readable JSON summary. +// +// go run ./scripts/sse-load -base http://127.0.0.1:18080 -duration 5m +// +// Credentials come from -user/-pass or CONTROL_USER/CONTROL_PASS. The tool +// exits non-zero when a client drops, an event is not delivered, or the p95 +// scenario latency exceeds 2 s: the contract's acceptance threshold. +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "math" + "net/http" + "os" + "os/signal" + "sort" + "strings" + "sync" + "syscall" + "time" +) + +const scenarioThreshold = 2 * time.Second + +// scenarioBodies rotate so consecutive changes differ in payload as well as +// version. +var scenarioBodies = []string{ + `{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0},"event":{"type":"market","uplift":0.2}}`, + `{"scenario":"rainy-saturday","weather":{"temp_delta_c":-3,"precip_mm":4},"event":{"type":"rain","uplift":-0.15}}`, + `{"scenario":"weekday-event","weather":{"temp_delta_c":0,"precip_mm":0},"event":{"type":"event","uplift":0.1}}`, + `{"scenario":"base","weather":{"temp_delta_c":0,"precip_mm":0},"event":{"type":"","uplift":0}}`, +} + +type scenarioEvent struct { + Version int `json:"version"` + At string `json:"at"` +} + +type client struct { + id int + + mu sync.Mutex + connected bool + receipts map[int]time.Time // scenario version -> first receipt + pings []time.Time + + readyOnce sync.Once +} + +func (c *client) setConnected(connected bool) { + c.mu.Lock() + c.connected = connected + c.mu.Unlock() +} + +func (c *client) isConnected() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.connected +} + +func (c *client) recordScenario(version int, at time.Time) { + c.mu.Lock() + if c.receipts == nil { + c.receipts = make(map[int]time.Time) + } + if _, exists := c.receipts[version]; !exists { + c.receipts[version] = at + } + c.mu.Unlock() +} + +func (c *client) recordPing(at time.Time) { + c.mu.Lock() + c.pings = append(c.pings, at) + c.mu.Unlock() +} + +func (c *client) receipt(version int) (time.Time, bool) { + c.mu.Lock() + defer c.mu.Unlock() + at, ok := c.receipts[version] + return at, ok +} + +func (c *client) pingCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.pings) +} + +func (c *client) signalReady(ready chan<- struct{}) { + c.readyOnce.Do(func() { + select { + case ready <- struct{}{}: + default: + } + }) +} + +type changeResult struct { + Version int `json:"version"` + PostAt time.Time `json:"post_at"` + Connected int `json:"connected_at_post"` + Received int `json:"received"` + Missing int `json:"missing"` + LatenciesMS []float64 `json:"-"` + P50MS float64 `json:"p50_ms"` + P95MS float64 `json:"p95_ms"` + P99MS float64 `json:"p99_ms"` + MaxMS float64 `json:"max_ms"` +} + +type summary struct { + StartedAt time.Time `json:"started_at"` + DurationS float64 `json:"duration_s"` + BaseURL string `json:"base_url"` + Clients int `json:"clients_configured"` + PeakConnected int `json:"clients_peak_connected"` + ConnectedAfter int `json:"clients_connected_after_warmup"` + + Drops int `json:"connection_drops"` + PostErrors int `json:"scenario_post_errors"` + + ScenarioChanges int `json:"scenario_changes"` + Received int `json:"scenario_events_received"` + Missing int `json:"scenario_events_missing"` + LatencyCount int `json:"scenario_latency_samples"` + LatencyP50MS float64 `json:"scenario_p50_ms"` + LatencyP95MS float64 `json:"scenario_p95_ms"` + LatencyP99MS float64 `json:"scenario_p99_ms"` + LatencyMaxMS float64 `json:"scenario_max_ms"` + Changes []changeResult `json:"changes"` + Ping pingSummary `json:"heartbeat"` + Errors []string `json:"errors"` + Passed bool `json:"passed"` + FailureReasons []string `json:"failure_reasons,omitempty"` +} + +type pingSummary struct { + Unique int `json:"unique_server_timestamps"` + MinIntervalMS float64 `json:"min_interval_ms"` + MedianIntervalMS float64 `json:"median_interval_ms"` + P95IntervalMS float64 `json:"p95_interval_ms"` + MaxIntervalMS float64 `json:"max_interval_ms"` + PerClientMin int `json:"per_client_min"` + PerClientMedian int `json:"per_client_median"` + PerClientMax int `json:"per_client_max"` +} + +type recorder struct { + mu sync.Mutex + clients []*client + connected int + peak int + drops int + postErrs int + errors []string + pingTimes map[time.Time]struct{} +} + +func (r *recorder) connectedNow() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.connected +} + +func (r *recorder) addConnected(delta int) { + r.mu.Lock() + r.connected += delta + if r.connected > r.peak { + r.peak = r.connected + } + r.mu.Unlock() +} + +func (r *recorder) addError(message string, drop bool) { + r.mu.Lock() + if drop { + r.drops++ + } else { + r.postErrs++ + } + if len(r.errors) < 40 { + r.errors = append(r.errors, message) + } + r.mu.Unlock() +} + +func (r *recorder) addPing(t time.Time) { + r.mu.Lock() + if r.pingTimes == nil { + r.pingTimes = make(map[time.Time]struct{}) + } + r.pingTimes[t] = struct{}{} + r.mu.Unlock() +} + +func (r *recorder) snapshot() (peak, drops, postErrs int, errs []string, pings []time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + peak, drops, postErrs = r.peak, r.drops, r.postErrs + errs = append([]string(nil), r.errors...) + pings = make([]time.Time, 0, len(r.pingTimes)) + for t := range r.pingTimes { + pings = append(pings, t) + } + sort.Slice(pings, func(i, j int) bool { return pings[i].Before(pings[j]) }) + return +} + +func main() { + base := flag.String("base", "http://127.0.0.1:18080", "Go API base URL") + clientsN := flag.Int("clients", 120, "concurrent SSE clients") + duration := flag.Duration("duration", 5*time.Minute, "load-test duration") + changeEvery := flag.Duration("changes", 30*time.Second, "interval between scenario POSTs; 0 disables") + warmup := flag.Duration("warmup", 10*time.Second, "max time to wait for all clients to connect") + user := flag.String("user", os.Getenv("CONTROL_USER"), "basic-auth user for POST /api/scenario") + pass := flag.String("pass", os.Getenv("CONTROL_PASS"), "basic-auth password for POST /api/scenario") + jsonOut := flag.String("json", "", "write the JSON summary to this path") + flag.Parse() + + if *clientsN < 1 { + fatalf("clients must be >= 1") + } + if *duration <= 0 { + fatalf("duration must be > 0") + } + if *changeEvery > 0 && (*user == "" || *pass == "") { + fatalf("POST /api/scenario needs -user/-pass or CONTROL_USER/CONTROL_PASS") + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithTimeout(ctx, *duration) + defer cancel() + + rec := &recorder{clients: make([]*client, *clientsN), pingTimes: make(map[time.Time]struct{})} + ready := make(chan struct{}, *clientsN) + started := time.Now() + + var wg sync.WaitGroup + for i := 0; i < *clientsN; i++ { + c := &client{id: i} + rec.clients[i] = c + wg.Add(1) + go func() { + defer wg.Done() + runClient(ctx, *base, c, rec, ready) + }() + } + + // Wait until every client has received the initial scenario event. + connectedAfter := waitReady(ctx, ready, *clientsN, *warmup) + fmt.Printf("clients connected: %d/%d after warmup\n", connectedAfter, *clientsN) + if connectedAfter < *clientsN { + fmt.Printf("warning: only %d/%d clients connected before warmup deadline\n", connectedAfter, *clientsN) + } + + changes := []changeResult{} + if *changeEvery > 0 { + changes = runChanges(ctx, *base, *user, *pass, *changeEvery, rec) + } + + <-ctx.Done() + cancel() + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + fmt.Println("warning: clients did not stop within 5s") + } + + elapsed := time.Since(started) + sum := buildSummary(rec, changes, *base, *clientsN, connectedAfter, started, elapsed) + printSummary(sum) + if *jsonOut != "" { + if err := writeJSON(*jsonOut, sum); err != nil { + fatalf("write %s: %v", *jsonOut, err) + } + fmt.Printf("json written to %s\n", *jsonOut) + } + if !sum.Passed { + os.Exit(1) + } +} + +func waitReady(ctx context.Context, ready <-chan struct{}, want int, timeout time.Duration) int { + timer := time.NewTimer(timeout) + defer timer.Stop() + count := 0 + for count < want { + select { + case <-ready: + count++ + case <-timer.C: + return count + case <-ctx.Done(): + return count + } + } + return count +} + +func runClient(ctx context.Context, base string, c *client, rec *recorder, ready chan<- struct{}) { + transport := &http.Transport{ + Proxy: nil, + DisableCompression: true, + MaxIdleConnsPerHost: 1, + IdleConnTimeout: 0, + } + httpClient := &http.Client{Transport: transport} + defer transport.CloseIdleConnections() + + streamURL := strings.TrimRight(base, "/") + "/api/stream" + failures := 0 + for ctx.Err() == nil { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, streamURL, nil) + if err != nil { + rec.addError(fmt.Sprintf("client %d: build request: %v", c.id, err), true) + c.signalReady(ready) + return + } + req.Header.Set("Accept", "text/event-stream") + resp, err := httpClient.Do(req) + if err != nil { + if ctx.Err() != nil { + return + } + failures++ + rec.addError(fmt.Sprintf("client %d: connect: %v", c.id, err), true) + c.signalReady(ready) + sleepRetry(ctx) + continue + } + if resp.StatusCode != http.StatusOK { + snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 128)) + _ = resp.Body.Close() + failures++ + rec.addError(fmt.Sprintf("client %d: status %d: %s", c.id, resp.StatusCode, strings.TrimSpace(string(snippet))), true) + c.signalReady(ready) + sleepRetry(ctx) + continue + } + + c.setConnected(true) + rec.addConnected(1) + readStream(ctx, resp.Body, c, rec, ready) + _ = resp.Body.Close() + c.setConnected(false) + rec.addConnected(-1) + + if ctx.Err() != nil { + return + } + failures++ + rec.addError(fmt.Sprintf("client %d: stream ended unexpectedly", c.id), true) + c.signalReady(ready) + sleepRetry(ctx) + } +} + +// readStream parses one SSE response. It returns when the body ends or the +// context is cancelled. +func readStream(ctx context.Context, body io.Reader, c *client, rec *recorder, ready chan<- struct{}) { + scanner := bufio.NewScanner(body) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + + eventName := "" + eventID := "" + var data []string + for ctx.Err() == nil && scanner.Scan() { + line := scanner.Text() + if line == "" { + now := time.Now() + switch eventName { + case "scenario": + version := 0 + var event scenarioEvent + if err := json.Unmarshal([]byte(strings.Join(data, "\n")), &event); err == nil { + version = event.Version + } + if version == 0 && eventID != "" { + _, _ = fmt.Sscanf(eventID, "%d", &version) + } + if version > 0 { + c.recordScenario(version, now) + c.signalReady(ready) + } + case "ping": + c.recordPing(now) + var payload struct { + T string `json:"t"` + } + if err := json.Unmarshal([]byte(strings.Join(data, "\n")), &payload); err == nil { + if t, err := time.Parse(time.RFC3339, payload.T); err == nil { + rec.addPing(t.UTC()) + } + } + } + eventName, eventID, data = "", "", nil + continue + } + switch { + case strings.HasPrefix(line, "event:"): + eventName = strings.TrimSpace(strings.TrimPrefix(line, "event:")) + case strings.HasPrefix(line, "id:"): + eventID = strings.TrimSpace(strings.TrimPrefix(line, "id:")) + case strings.HasPrefix(line, "data:"): + data = append(data, strings.TrimPrefix(line, "data:")) + } + } +} + +func runChanges(ctx context.Context, base, user, pass string, every time.Duration, rec *recorder) []changeResult { + client := &http.Client{Timeout: 10 * time.Second} + scenarioURL := strings.TrimRight(base, "/") + "/api/scenario" + + var out []changeResult + ticker := time.NewTicker(every) + defer ticker.Stop() + + for i := 0; ; i++ { + select { + case <-ctx.Done(): + return out + case <-ticker.C: + } + body := scenarioBodies[i%len(scenarioBodies)] + postAt := time.Now() + version, err := postScenario(ctx, client, scenarioURL, user, pass, body) + if err != nil { + if ctx.Err() != nil { + return out + } + rec.addError("scenario POST: "+err.Error(), false) + continue + } + result := changeResult{ + Version: version, + PostAt: postAt, + Connected: rec.connectedNow(), + } + + // Wait up to 2s for every currently connected client to receive it. + deadline := time.Now().Add(2 * time.Second) + for { + result.Received = 0 + for _, c := range rec.clients { + if !c.isConnected() { + continue + } + if _, ok := c.receipt(version); ok { + result.Received++ + } + } + result.Missing = result.Connected - result.Received + if result.Missing <= 0 || time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + + for _, c := range rec.clients { + if !c.isConnected() { + continue + } + if at, ok := c.receipt(version); ok { + result.LatenciesMS = append(result.LatenciesMS, + float64(at.Sub(postAt).Microseconds())/1000.0) + } + } + result.P50MS, result.P95MS, result.P99MS, result.MaxMS = percentiles(result.LatenciesMS) + out = append(out, result) + fmt.Printf("scenario v%d: %d/%d clients p50=%.1fms p95=%.1fms max=%.1fms\n", + version, result.Received, result.Connected, result.P50MS, result.P95MS, result.MaxMS) + } +} + +func postScenario(ctx context.Context, client *http.Client, url, user, pass, body string) (int, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBufferString(body)) + if err != nil { + return 0, err + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(user, pass) + resp, err := client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var parsed struct { + Version int `json:"version"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return 0, fmt.Errorf("decode response: %w", err) + } + if parsed.Version <= 0 { + return 0, errors.New("response has no version") + } + return parsed.Version, nil +} + +func buildSummary(rec *recorder, changes []changeResult, base string, clientsN, connectedAfter int, started time.Time, elapsed time.Duration) summary { + peak, drops, postErrs, errs, pingTimes := rec.snapshot() + + sum := summary{ + StartedAt: started.UTC(), + DurationS: math.Round(elapsed.Seconds()*1000) / 1000, + BaseURL: base, + Clients: clientsN, + PeakConnected: peak, + ConnectedAfter: connectedAfter, + Drops: drops, + PostErrors: postErrs, + ScenarioChanges: len(changes), + Changes: changes, + Errors: errs, + } + var all []float64 + for _, change := range changes { + all = append(all, change.LatenciesMS...) + sum.Received += change.Received + sum.Missing += change.Missing + } + sum.LatencyCount = len(all) + sum.LatencyP50MS, sum.LatencyP95MS, sum.LatencyP99MS, sum.LatencyMaxMS = percentiles(all) + sum.Ping = buildPingSummary(pingTimes, rec.clients) + + // Pass criteria from CONTRACT.md ยง6: 120 clients for 5 minutes, no dropped + // connections, p95 scenario latency below 2 s. + if peak < clientsN { + sum.FailureReasons = append(sum.FailureReasons, fmt.Sprintf("peak connections %d < %d", peak, clientsN)) + } + if drops > 0 { + sum.FailureReasons = append(sum.FailureReasons, fmt.Sprintf("%d dropped/reconnected connections", drops)) + } + if postErrs > 0 { + sum.FailureReasons = append(sum.FailureReasons, fmt.Sprintf("%d scenario POST errors", postErrs)) + } + if sum.Missing > 0 { + sum.FailureReasons = append(sum.FailureReasons, fmt.Sprintf("%d scenario events missing", sum.Missing)) + } + if len(all) > 0 && sum.LatencyP95MS > scenarioThreshold.Seconds()*1000 { + sum.FailureReasons = append(sum.FailureReasons, fmt.Sprintf("scenario p95 %.1fms > 2000ms", sum.LatencyP95MS)) + } + if sum.Ping.P95IntervalMS > 17_000 { + sum.FailureReasons = append(sum.FailureReasons, fmt.Sprintf("heartbeat p95 interval %.0fms > 17000ms", sum.Ping.P95IntervalMS)) + } + sum.Passed = len(sum.FailureReasons) == 0 + return sum +} + +func buildPingSummary(pingTimes []time.Time, clients []*client) pingSummary { + out := pingSummary{Unique: len(pingTimes)} + if len(pingTimes) > 1 { + intervals := make([]float64, 0, len(pingTimes)-1) + for i := 1; i < len(pingTimes); i++ { + intervals = append(intervals, float64(pingTimes[i].Sub(pingTimes[i-1]).Microseconds())/1000.0) + } + out.MinIntervalMS = minFloat(intervals) + out.MedianIntervalMS = percentile(intervals, 50) + out.P95IntervalMS = percentile(intervals, 95) + out.MaxIntervalMS = maxFloat(intervals) + } + counts := make([]int, 0, len(clients)) + for _, c := range clients { + counts = append(counts, c.pingCount()) + } + if len(counts) > 0 { + sort.Ints(counts) + out.PerClientMin = counts[0] + out.PerClientMedian = counts[len(counts)/2] + out.PerClientMax = counts[len(counts)-1] + } + return out +} + +func percentiles(values []float64) (p50, p95, p99, max float64) { + if len(values) == 0 { + return 0, 0, 0, 0 + } + return percentile(values, 50), percentile(values, 95), percentile(values, 99), maxFloat(values) +} + +// percentile uses the nearest-rank method on a sorted copy. +func percentile(values []float64, p float64) float64 { + if len(values) == 0 { + return 0 + } + sorted := append([]float64(nil), values...) + sort.Float64s(sorted) + rank := int(math.Ceil(p/100*float64(len(sorted)))) - 1 + if rank < 0 { + rank = 0 + } + if rank >= len(sorted) { + rank = len(sorted) - 1 + } + return math.Round(sorted[rank]*10) / 10 +} + +func minFloat(values []float64) float64 { + m := values[0] + for _, v := range values[1:] { + if v < m { + m = v + } + } + return m +} + +func maxFloat(values []float64) float64 { + m := values[0] + for _, v := range values[1:] { + if v > m { + m = v + } + } + return m +} + +func sleepRetry(ctx context.Context) { + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +func printSummary(sum summary) { + fmt.Println() + fmt.Println("=== SSE load test ===") + fmt.Printf("duration: %.1fs\n", sum.DurationS) + fmt.Printf("clients: %d configured, %d peak connected, %d after warmup\n", + sum.Clients, sum.PeakConnected, sum.ConnectedAfter) + fmt.Printf("connections: %d drops/reconnects, %d scenario POST errors\n", sum.Drops, sum.PostErrors) + fmt.Printf("scenario changes: %d (%d events received, %d missing)\n", + sum.ScenarioChanges, sum.Received, sum.Missing) + fmt.Printf("scenario latency: p50=%.1fms p95=%.1fms p99=%.1fms max=%.1fms (n=%d)\n", + sum.LatencyP50MS, sum.LatencyP95MS, sum.LatencyP99MS, sum.LatencyMaxMS, sum.LatencyCount) + fmt.Printf("heartbeat: %d server pings, interval min=%.0fms median=%.0fms p95=%.0fms max=%.0fms\n", + sum.Ping.Unique, sum.Ping.MinIntervalMS, sum.Ping.MedianIntervalMS, + sum.Ping.P95IntervalMS, sum.Ping.MaxIntervalMS) + fmt.Printf("heartbeat/client: min=%d median=%d max=%d\n", + sum.Ping.PerClientMin, sum.Ping.PerClientMedian, sum.Ping.PerClientMax) + if len(sum.Errors) > 0 { + fmt.Println("errors:") + for _, err := range sum.Errors { + fmt.Printf(" - %s\n", err) + } + } + if sum.Passed { + fmt.Println("result: PASS") + } else { + fmt.Println("result: FAIL") + for _, reason := range sum.FailureReasons { + fmt.Printf(" - %s\n", reason) + } + } +} + +func writeJSON(path string, sum summary) error { + raw, err := json.MarshalIndent(sum, "", " ") + if err != nil { + return err + } + raw = append(raw, '\n') + return os.WriteFile(path, raw, 0o644) +} + +func fatalf(format string, args ...any) { + fmt.Fprintf(os.Stderr, "sse-load: "+format+"\n", args...) + os.Exit(2) +} diff --git a/scripts/sse-load/main_test.go b/scripts/sse-load/main_test.go new file mode 100644 index 0000000..4857064 --- /dev/null +++ b/scripts/sse-load/main_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "math" + "testing" + "time" +) + +func TestPercentileNearestRank(t *testing.T) { + values := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + cases := []struct { + p float64 + want float64 + }{ + {50, 5}, + {95, 10}, + {99, 10}, + {0, 1}, + {100, 10}, + } + for _, tc := range cases { + if got := percentile(values, tc.p); got != tc.want { + t.Fatalf("percentile(%v, %v)=%v, want %v", values, tc.p, got, tc.want) + } + } + if got := percentile(nil, 95); got != 0 { + t.Fatalf("empty percentile=%v, want 0", got) + } + + p50, p95, p99, max := percentiles(values) + if p50 != 5 || p95 != 10 || p99 != 10 || max != 10 { + t.Fatalf("percentiles=(%v,%v,%v,%v)", p50, p95, p99, max) + } +} + +func TestPercentileDoesNotMutateInput(t *testing.T) { + values := []float64{3, 1, 2} + _ = percentile(values, 50) + if values[0] != 3 || values[1] != 1 || values[2] != 2 { + t.Fatalf("input was reordered: %v", values) + } +} + +func TestBuildPingSummary(t *testing.T) { + start := time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC) + var pings []time.Time + for i := 0; i < 5; i++ { + pings = append(pings, start.Add(time.Duration(i)*15*time.Second)) + } + clients := []*client{ + {pings: pings[:3]}, + {pings: pings[:4]}, + {pings: pings}, + } + got := buildPingSummary(pings, clients) + if got.Unique != 5 { + t.Fatalf("unique=%d, want 5", got.Unique) + } + if math.Abs(got.MedianIntervalMS-15000) > 0.1 { + t.Fatalf("median interval=%v, want 15000", got.MedianIntervalMS) + } + if got.MaxIntervalMS != 15000 || got.MinIntervalMS != 15000 { + t.Fatalf("interval range=[%v,%v], want [15000,15000]", got.MinIntervalMS, got.MaxIntervalMS) + } + if got.PerClientMin != 3 || got.PerClientMedian != 4 || got.PerClientMax != 5 { + t.Fatalf("per-client=(%d,%d,%d), want (3,4,5)", + got.PerClientMin, got.PerClientMedian, got.PerClientMax) + } +} + +func TestSummaryPassAndFail(t *testing.T) { + started := time.Now() + makeRecorder := func(peak, drops, postErrs int) *recorder { + return &recorder{ + peak: peak, + drops: drops, + postErrs: postErrs, + pingTimes: make(map[time.Time]struct{}), + } + } + changes := []changeResult{{ + Version: 2, + Connected: 120, + Received: 120, + LatenciesMS: []float64{4, 6, 8}, + }} + pass := buildSummary(makeRecorder(120, 0, 0), changes, "http://example", 120, 120, started, 5*time.Minute) + if !pass.Passed { + t.Fatalf("expected pass, reasons=%v", pass.FailureReasons) + } + + fail := buildSummary(makeRecorder(119, 1, 1), changes, "http://example", 120, 120, started, 5*time.Minute) + if fail.Passed { + t.Fatal("expected failure for low peak and drops") + } + if len(fail.FailureReasons) < 3 { + t.Fatalf("failure reasons=%v, want at least 3", fail.FailureReasons) + } +} + +func TestSlowLatencyFailsThreshold(t *testing.T) { + rec := &recorder{peak: 120, pingTimes: make(map[time.Time]struct{})} + changes := []changeResult{{ + Version: 3, + Connected: 120, + Received: 120, + LatenciesMS: []float64{2500}, + }} + sum := buildSummary(rec, changes, "http://example", 120, 120, time.Now(), 5*time.Minute) + if sum.Passed { + t.Fatal("expected p95 latency over 2s to fail") + } +} + +func TestScenarioReceiptKeepsFirst(t *testing.T) { + c := &client{} + first := time.Unix(100, 0) + c.recordScenario(4, first) + c.recordScenario(4, first.Add(time.Second)) + got, ok := c.receipt(4) + if !ok || !got.Equal(first) { + t.Fatalf("receipt=%v ok=%v, want %v", got, ok, first) + } +}