From b98a1bab42e0863348c4226fc3a8dc3617fcfd40 Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 08:00:58 +0100 Subject: [PATCH] feat: prepare v1.1 deploy and presenter load test - build and push v1.1 images from the service main worktrees - switch manifests to :v1.1; add web control Secret refs - configure API BOOTSTRAP_REFRESH=60s and remove unused DATA_DIR - add k6 presenter scenario with a mid-run control POST - require control credentials in make load; update runbook and drills --- Makefile | 32 ++++++--- README.md | 20 +++--- deploy/k8s/00-config.yaml | 4 +- deploy/k8s/10-ml.yaml | 6 +- deploy/k8s/20-api.yaml | 6 +- deploy/k8s/30-web.yaml | 23 ++++++- docs/deploy.md | 129 +++++++++++++++++++++---------------- docs/failure-drills.md | 11 +++- docs/load-test.md | 72 +++++++++++++-------- k6/sse.js | 67 ++++++++++++++++++- scripts/check_manifests.py | 9 ++- scripts/gate1-check.sh | 2 +- 12 files changed, 264 insertions(+), 117 deletions(-) diff --git a/Makefile b/Makefile index 79310f5..9f00afa 100644 --- a/Makefile +++ b/Makefile @@ -11,20 +11,20 @@ PYTHON ?= python3 REGISTRY ?= git.noelsabia.dev OWNER ?= noel -TAG ?= gate1 +TAG ?= v1.1 WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG) API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG) ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG) -# Service repos are bare Git repos beside this one, each with per-task worktrees: -# Hackathon_2026/web-nuxt.git/feat-web-bootstrap -# Hackathon_2026/api-go.git/feat-api-bootstrap -# Hackathon_2026/ML-py.git/feat-ml-bootstrap +# Release images are built from the service integration branches: +# Hackathon_2026/web-nuxt.git/main +# Hackathon_2026/api-go.git/main +# Hackathon_2026/ML-py.git/main # Override any *_DIR on the command line when the layout differs. SERVICE_REPOS ?= $(abspath $(ROOT)/../..) -WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/main))) -API_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/main))) -ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main))) +WEB_DIR ?= $(SERVICE_REPOS)/web-nuxt.git/main +API_DIR ?= $(SERVICE_REPOS)/api-go.git/main +ML_DIR ?= $(SERVICE_REPOS)/ML-py.git/main WEB_URL ?= https://pulse.noelsabia.dev API_URL ?= https://api-pulse.noelsabia.dev @@ -36,6 +36,11 @@ SKIP_ML ?= 0 SSE_VUS ?= 120 SSE_HOLD_PINGS ?= 20 SSE_MAX_DURATION ?= 6m +CONTROL_USER ?= +CONTROL_PASS ?= +PRESENTER_SCENARIO ?= rainy-saturday +PRESENTER_AT ?= 2m30s +PRESENTER_TIMEOUT ?= 10s K6 ?= K6_BIN ?= $(ROOT)/k6/k6-sse # xk6-sse v0.2.0 exposes client.on/client.close correctly on k6 v2.0.x but not @@ -153,8 +158,13 @@ smoke-local: ## Smoke the three local dev servers SMOKE_SSE="$(SMOKE_SSE)" SKIP_ML="$(SKIP_ML)" \ "$(ROOT)/scripts/smoke.sh" -load: ## Run the 120-client / 5-minute SSE load test +load: ## Run the 120-client / 5-minute SSE load test + one presenter POST @mkdir -p "$(K6_RESULTS)" + @if [ -z "$(CONTROL_USER)" ] || [ -z "$(CONTROL_PASS)" ]; then \ + echo "CONTROL_USER and CONTROL_PASS are required for the presenter scenario."; \ + echo "Run: set -a; source ~/.config/pulsecast/control-auth.env; set +a"; \ + exit 1; \ + fi @set -e; \ if [ -n "$(K6)" ]; then \ runner="$(K6)"; \ @@ -166,7 +176,9 @@ load: ## Run the 120-client / 5-minute SSE load test echo "No k6 binary. Run 'make k6-build' once (needs Go), then retry."; \ exit 1; \ fi; \ - export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \ + export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" SSE_MAX_DURATION="$(SSE_MAX_DURATION)" \ + CONTROL_USER="$(CONTROL_USER)" CONTROL_PASS="$(CONTROL_PASS)" \ + PRESENTER_SCENARIO="$(PRESENTER_SCENARIO)" PRESENTER_AT="$(PRESENTER_AT)" PRESENTER_TIMEOUT="$(PRESENTER_TIMEOUT)"; \ stamp=$$(date +%Y%m%dT%H%M%S); \ echo "Using $$runner"; \ "$$runner" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)" diff --git a/README.md b/README.md index c88db03..2077d59 100644 --- a/README.md +++ b/README.md @@ -45,17 +45,21 @@ hostname, and verification steps. ## Gate 1 status -- Manifests validate read-only; `make gate1-check` is green except for pending - human prerequisites. -- All three images build and pass local manifest-equivalent runtime checks +- Static `make gate1-check STATIC_ONLY=1`/CI passes. The live check correctly + flags the cluster's v1.0 images until the reviewed v1.1 branch is applied. +- All three v1.1 images build and pass local manifest-equivalent runtime checks (read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker. - Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from `git.noelsabia.dev/noel/hello:gate1`; the Service returned HTTP 200. -- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes; - `https://pulse.noelsabia.dev` returns 200 from hello with valid TLS. The API - hostname is `api-pulse.noelsabia.dev` because Universal SSL only covers - `*.noelsabia.dev`. -- Still needed: `control-auth` Secret, image push, deploy, and public smoke. +- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes. + `pulse` still points at the temporary hello Service until the v1.1 cutover. + The API hostname is `api-pulse.noelsabia.dev` because Universal SSL only + covers `*.noelsabia.dev`. +- v1.1 images are built from the service `main` worktrees, locally checked, + and pushed. The cluster still runs the previous v1.0 images until this branch + is reviewed/merged and explicitly applied. +- The v1.1 branch adds web control credentials, API bootstrap config, and the + k6 presenter scenario; load-test results are recorded after the approved run. ## CI diff --git a/deploy/k8s/00-config.yaml b/deploy/k8s/00-config.yaml index 0292784..da0a3f3 100644 --- a/deploy/k8s/00-config.yaml +++ b/deploy/k8s/00-config.yaml @@ -42,8 +42,10 @@ metadata: app.kubernetes.io/part-of: pulsecast data: PORT: "8080" - DATA_DIR: "/app/data" FASTAPI_URL: "http://ml:8000" + # v1.1 fetches FastAPI /bootstrap at startup and every BOOTSTRAP_REFRESH; + # there is no baked data directory and no DATA_DIR env. + BOOTSTRAP_REFRESH: "60s" ML_TIMEOUT: "1.5s" SSE_HEARTBEAT: "15s" --- diff --git a/deploy/k8s/10-ml.yaml b/deploy/k8s/10-ml.yaml index 2718ddb..64dc621 100644 --- a/deploy/k8s/10-ml.yaml +++ b/deploy/k8s/10-ml.yaml @@ -35,9 +35,9 @@ spec: fsGroup: 10001 containers: - name: ml - image: git.noelsabia.dev/noel/pulsecast-ml:gate1 - # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. - # Switch to an immutable tag + IfNotPresent at deploy freeze. + image: git.noelsabia.dev/noel/pulsecast-ml:v1.1 + # v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible. + # Switch back to IfNotPresent once the tag is frozen. imagePullPolicy: Always ports: - name: http diff --git a/deploy/k8s/20-api.yaml b/deploy/k8s/20-api.yaml index dcf03f0..241606e 100644 --- a/deploy/k8s/20-api.yaml +++ b/deploy/k8s/20-api.yaml @@ -36,9 +36,9 @@ spec: fsGroup: 65532 containers: - name: api - image: git.noelsabia.dev/noel/pulsecast-api:gate1 - # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. - # Switch to an immutable tag + IfNotPresent at deploy freeze. + image: git.noelsabia.dev/noel/pulsecast-api:v1.1 + # v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible. + # Switch back to IfNotPresent once the tag is frozen. imagePullPolicy: Always ports: - name: http diff --git a/deploy/k8s/30-web.yaml b/deploy/k8s/30-web.yaml index 672032a..969e87d 100644 --- a/deploy/k8s/30-web.yaml +++ b/deploy/k8s/30-web.yaml @@ -39,9 +39,9 @@ spec: containers: - name: web # Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`. - image: git.noelsabia.dev/noel/pulsecast-web:gate1 - # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. - # Switch to an immutable tag + IfNotPresent at deploy freeze. + image: git.noelsabia.dev/noel/pulsecast-web:v1.1 + # v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible. + # Switch back to IfNotPresent once the tag is frozen. imagePullPolicy: Always ports: - name: http @@ -58,6 +58,23 @@ spec: # Nuxt/Nitro may use a temp dir; keep it on the writable emptyDir. - name: NITRO_TEMP_DIR value: /tmp + # The web server validates /control and POST /api/scenario itself + # and attaches these same credentials to Go control calls. + - name: CONTROL_USER + valueFrom: + secretKeyRef: + name: control-auth + key: CONTROL_USER + - name: CONTROL_PASS + valueFrom: + secretKeyRef: + name: control-auth + key: CONTROL_PASS + - name: CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: control-auth + key: CONTROL_TOKEN resources: requests: cpu: 100m diff --git a/docs/deploy.md b/docs/deploy.md index 7edf019..c00fc6e 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -24,7 +24,7 @@ Gate 1 passes only when **all** of these are true: Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest dry-run, or a container that only runs on your laptop. -## Current external status (2026-11-14 ~06:50 CET) +## Current external status (2026-11-14 ~08:00 CET) Re-check before trusting this snapshot: @@ -33,8 +33,8 @@ Re-check before trusting this snapshot: Open-Meteo, and `git.noelsabia.dev`. - **Tunnel active:** `pulsecast-gate1` has routes for `pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`; `cloudflared` is `1/1` with four HTTP/2 - connections, and `https://pulse.noelsabia.dev` returns 200 from hello with - valid TLS. `api-pulse` returns 502 until the API is deployed. + connections. `api-pulse` serves the deployed API; `pulse` still points at + the temporary hello Service until the web cutover. - **DNS works:** public resolvers return Cloudflare IPs for `pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`. A local `systemd-resolved` negative cache can be cleared with @@ -42,20 +42,19 @@ Re-check before trusting this snapshot: - **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only `*.noelsabia.dev`; a two-level `api.pulse.noelsabia.dev` is not covered by that certificate. -- **Gitea pull is proven (2026-11-14 ~06:16 CET):** `gitea-regcred` exists and - the cluster pulled `git.noelsabia.dev/noel/hello:gate1` in 120 ms; the hello - pod is `1/1 Running` and its Service returned HTTP 200. The Gitea owner is - `noel`. +- **Gitea pull is proven:** `gitea-regcred` exists; hello and the v1.0 service + images pulled from Gitea. The Gitea owner is `noel`. +- **v1.1 images pushed:** web/api/ml `:v1.1` were built from the service + `main` worktrees, locally exercised under numeric UIDs/read-only root, and + pushed. The cluster still runs the previous v1.0 images until the v1.1 + branch is reviewed/merged and explicitly applied. +- **All three Secrets exist:** `gitea-regcred`, `cloudflared`, `control-auth`. - **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the `cloudflared` manifest uses `--protocol http2` instead of relying on unverified UDP. - **Numeric UIDs are required:** a non-numeric image user breaks `runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in `deploy/k8s/` sets explicit numeric `runAsUser`/`runAsGroup`/`fsGroup`. -- **Repo status (2026-11-14 ~06:50 CET):** all three images build and pass - local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp` - tmpfs). Remaining steps are the image push, `control-auth`, deploy, and - public smoke. ## What is possible now vs what is blocked @@ -63,12 +62,10 @@ Re-check before trusting this snapshot: |---|---|---| | Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) | | Gitea push login + hello-world push | human | **done**; hello pull and Service verified | -| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` + `cloudflared` **done**; `control-auth` pending | -| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active, public hello 200 | -| `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally | -| `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally | -| `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally | -| Full deploy + smoke + Gate 1 sign-off | human | **ready once the tunnel/secret/image steps are done** | +| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **done** | +| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active; `pulse` still points at hello | +| `api`, `ml`, `web` images | human | **v1.1 pushed**; local manifest-equivalent checks pass | +| v1.1 apply + smoke + presenter k6 + drills | human | **blocked on branch review/merge** | `make build` and `make push` cover all three services; no Dockerfile is missing now. Do not deploy placeholder images under the real service names. @@ -86,7 +83,7 @@ export NS="fch26-onemanarmy" export CTX="fch26-onemanarmy-sa-context" export REGISTRY="git.noelsabia.dev" export OWNER="noel" -export TAG="gate1" +export TAG="v1.1" chmod 600 "$KUBECONFIG" ``` @@ -147,7 +144,7 @@ Run top to bottom. Stop at the first failure and use - [ ] 5. `cloudflared` Secret created; logs show `Registered tunnel connection`. - [ ] 6. Public hello route returns HTTP 200 (proves tunnel + DNS + TLS). - [ ] 7. `control-auth` Secret created. -- [ ] 8. All three images built, tagged `gate1`, and pushed. +- [ ] 8. All three v1.1 images built and pushed. - [ ] 9. `make validate` and `make gate1-check` pass. - [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready. - [ ] 11. `make smoke SMOKE_SSE=1` passes. @@ -411,10 +408,10 @@ kubectl --context "$CTX" -n "$NS" describe pod -l app.kubernetes.io/name=cloudfl ## Step 4 — control-auth Secret -The API reads `CONTROL_USER` and `CONTROL_PASS` for browser control auth. -Both API and ML also need the same `CONTROL_TOKEN`: the API forwards it as a -Bearer token to FastAPI's internal `/train` routes. Generate both locally; -never commit them. +The web server validates `/control` and POST `/api/scenario` with +`CONTROL_USER`/`CONTROL_PASS`; web, API, and ML also need the same +`CONTROL_TOKEN` (the API forwards it as a Bearer token to FastAPI's internal +`/train` routes). Generate both locally; never commit them. ```bash read -rsp 'Control password: ' CONTROL_PASS; echo @@ -438,9 +435,9 @@ creates nothing. Check whether the Dockerfiles exist: ```bash -test -f ../../web-nuxt.git/feat-web-bootstrap/Dockerfile && echo web-ok || echo web-blocked -test -f ../../api-go.git/feat-api-bootstrap/Dockerfile && echo api-ok || echo api-blocked -test -f ../../ML-py.git/feat-ml-bootstrap/Dockerfile && echo ml-ok || echo ml-blocked +test -f ../../web-nuxt.git/main/Dockerfile && echo web-ok || echo web-blocked +test -f ../../api-go.git/main/Dockerfile && echo api-ok || echo api-blocked +test -f ../../ML-py.git/main/Dockerfile && echo ml-ok || echo ml-blocked ``` Expected **now** (all three Dockerfiles landed on 2026-11-14): @@ -457,18 +454,20 @@ Build all three: make build ``` -All three images were verified locally with the manifest's runtime +All three v1.1 images were verified locally with the manifest's runtime constraints: - API: distroless `static:nonroot`, `--user 65532:65532`, read-only root, - `/tmp` tmpfs; `/api/healthz` and `/api/readyz` returned 200. -- ML: image `USER app` is non-numeric, but the manifest overrides it with - `runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/healthz` and - `/readyz` returned 200 and `/meta` loaded the baked model. -- Web: image `USER node` is non-numeric, but the manifest overrides it with - `runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/` returned - HTTP 200 and the runtime `NUXT_PUBLIC_API_BASE` appeared in the HTML. -- No image was pushed or deployed by the infra agent. + `/tmp` tmpfs; `/api/healthz` and `/api/readyz` returned 200, and the API + loaded ML `/bootstrap` into memory. +- ML: image `USER 10001:10001`; with read-only root and `/tmp` tmpfs, + `/healthz` and `/readyz` returned 200, `/bootstrap` returned 8 sensors, and + `/meta` showed `mae=3.32`, `wape=27.5`, `r2=0.777`. +- Web: image `USER 10001:10001`; with read-only root and `/tmp` tmpfs, `/`, + `/stage`, and `/live` returned 200; `/control` returned 401 anonymous and + 200 with control auth; the same-origin SSE proxy emitted events. +- v1.1 was pushed to Gitea. The cluster is not updated until this branch is + reviewed/merged and explicitly applied. Push: @@ -482,9 +481,9 @@ repeat the push login from Step 1. The Makefile auto-detects the service worktrees: ```text -../../web-nuxt.git/feat-web-bootstrap -../../api-go.git/feat-api-bootstrap -../../ML-py.git/feat-ml-bootstrap +../../web-nuxt.git/main +../../api-go.git/main +../../ML-py.git/main ``` Override when the layout differs: @@ -512,9 +511,9 @@ Each service image must: | Service | Image | Port | UID:GID | Health paths | |---|---|---|---|---| -| web | `git.noelsabia.dev/noel/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` | -| api | `git.noelsabia.dev/noel/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` | -| ml | `git.noelsabia.dev/noel/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` | +| web | `git.noelsabia.dev/noel/pulsecast-web:v1.1` | 3000 | `10001:10001` | `/`, `/stage`, `/live`, `/control` | +| api | `git.noelsabia.dev/noel/pulsecast-api:v1.1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` | +| ml | `git.noelsabia.dev/noel/pulsecast-ml:v1.1` | 8000 | `10001:10001` | `/healthz`, `/readyz`, `/bootstrap` | The manifests set `runAsUser`/`runAsGroup`/`fsGroup` explicitly, so the image's named user is irrelevant; the files must be readable by that numeric UID. @@ -522,17 +521,19 @@ named user is irrelevant; the files must be readable by that numeric UID. Dockerfile genuinely needs another UID, change the manifest and this table together, and keep the value numeric. -`api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`, -`CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and -`ML_TIMEOUT`. -`web` must have `NUXT_PUBLIC_API_BASE=https://api-pulse.noelsabia.dev` baked in -or read it at runtime from the ConfigMap; the browser must use the public API -hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`, -`PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and -`CONTROL_TOKEN`; its image CMD binds `0.0.0.0:8000`. The env names in the -manifests are the integration point: change them here first if a service needs -a different name. The ML image currently uses a named `app` user; the manifest -overrides it with numeric UID 10001, which was verified locally. +`api` v1.1 reads `FASTAPI_URL=http://ml:8000`, `BOOTSTRAP_REFRESH=60s`, +`BOOTSTRAP_TIMEOUT`, `PORT`, `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, +`SSE_HEARTBEAT`, and `ML_TIMEOUT`. It has **no baked data and no `DATA_DIR`**; +it fetches ML `/bootstrap` into memory and refreshes it. +`web` proxies Go and SSE server-side using `NUXT_API_INTERNAL_BASE=http://api:8080`; +it also needs `CONTROL_USER`, `CONTROL_PASS`, and `CONTROL_TOKEN` to protect +`/control` and POST `/api/scenario`. `NUXT_PUBLIC_API_BASE` and the public SSE +URL are display/future configuration; the browser calls the Nuxt origin. +`ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`, `PULSECAST_WEATHER`, +`PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and `CONTROL_TOKEN`; its image CMD +binds `0.0.0.0:8000`. The env names in the manifests are the integration +point: change them here first if a service needs a different name. All three +images use numeric UIDs: web/ml `10001:10001`, API `65532:65532`. ## Step 6 — prove public HTTPS with hello @@ -623,7 +624,7 @@ Expected: `web`, `api`, `ml`, and `cloudflared` each `1/1`; Services `web`, ### Re-pushing the same tag -The manifests use `imagePullPolicy: Always` while the tag stays `gate1`. After +The manifests use `imagePullPolicy: Always` while the tag stays `v1.1`. After rebuilding and pushing the same tag, force a re-pull: ```bash @@ -680,6 +681,26 @@ The SSE output must start with `retry: 3000`, contain a `scenario` event immediately, and show a `ping` within 20 seconds. If events are delayed through Cloudflare, see the SSE row in Troubleshooting. +### v1.1 product checks + +```bash +curl -fsS https://api-pulse.noelsabia.dev/api/sensors | jq 'length' +curl -fsS https://api-pulse.noelsabia.dev/api/meta \ + | jq '{data_version,model_version,metrics,cache}' +curl -fsS -o /dev/null -w 'stage %{http_code}\n' https://pulse.noelsabia.dev/stage +curl -fsS -o /dev/null -w 'live %{http_code}\n' https://pulse.noelsabia.dev/live +curl -sS -o /dev/null -w 'control anon %{http_code}\n' https://pulse.noelsabia.dev/control +set -a; source ~/.config/pulsecast/control-auth.env; set +a +curl -sS -u "$CONTROL_USER:$CONTROL_PASS" -o /dev/null \ + -w 'control auth %{http_code}\n' https://pulse.noelsabia.dev/control +curl -sS -o /dev/null -w 'plan %{time_total}s\n' \ + 'https://api-pulse.noelsabia.dev/api/staffing?sensor=hn-stadtgalerie-01&scenario=base&standard=standard' +``` + +Expected: **8** sensors; metrics `mae=3.32`, `wape=27.5`, `r2=0.777` with a +`cache` object; `/stage` and `/live` 200; `/control` 401 anonymous and 200 +with control auth; plan under 2 seconds. + In-cluster connectivity (optional; **creates and removes one short-lived pod**, so the human runs it): @@ -738,7 +759,7 @@ you need to reproduce the exact deploy. | `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. | | `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. | | Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. | -| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. | +| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS http://ml:8000/bootstrap` from inside the namespace; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | v1.1 has no baked data. It needs ML reachable at `FASTAPI_URL` and a successful `/bootstrap`; check `BOOTSTRAP_REFRESH` and the ML readiness. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. | | `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. | | `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. | | SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. | diff --git a/docs/failure-drills.md b/docs/failure-drills.md index d4bf299..720c495 100644 --- a/docs/failure-drills.md +++ b/docs/failure-drills.md @@ -7,8 +7,9 @@ drill. ## 1. FastAPI down (cached fallback) -Goal: the Go API keeps serving forecasts from baked JSON with `"cached": true` -and `/api/readyz` stays 200. FastAPI being down must not fail API readiness. +Goal: the Go API keeps serving its last good in-memory `/bootstrap` snapshot +with `"cached": true` and `/api/readyz` stays 200. FastAPI being down must not +fail API readiness after the snapshot has loaded once. ```bash # Baseline @@ -26,6 +27,9 @@ curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01 # Recover kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=1 kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/ml --timeout=180s +# The API refreshes within BOOTSTRAP_REFRESH=60s; confirm the cache is fresh again. +sleep 65 +curl -sS https://api-pulse.noelsabia.dev/api/meta | jq '.cache' ``` Pass: no 5xx from `/api/forecast`, `cached` flips to `true` within the 1.5 s @@ -65,7 +69,8 @@ the tunnel token means updating the Secret and restarting the Deployment. Goal: a single API pod restart causes a short 502 window, not a stuck UI. SSE clients reconnect with `retry: 3000`; scenario state is in memory and resets to -the default, which is documented. +the default. The restarted API re-fetches ML `/bootstrap` before `/api/readyz` +turns 200; keep ML up during this drill. ```bash # Watch from two terminals: diff --git a/docs/load-test.md b/docs/load-test.md index 4ecd75d..3ea386a 100644 --- a/docs/load-test.md +++ b/docs/load-test.md @@ -1,7 +1,9 @@ # k6 load test `k6/sse.js` holds 120 concurrent Server-Sent Events connections for 5 minutes -and reports connection count plus p95 event latency. +and reports connection count plus p95 event latency. A separate `presenter` +scenario POSTs one scenario change at 2m30s so the test measures event +propagation through Cloudflare, not just heartbeat latency. ## Why a custom k6 binary @@ -36,62 +38,77 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension. You should see `retry: 3000` and a `scenario` event immediately, then a `ping` every 15 s. -2. Run the load test: +2. Load the control credentials and run the load test. The presenter scenario + requires `CONTROL_USER` and `CONTROL_PASS`; `make load` refuses to start + without them. ```bash + set -a + source ~/.config/pulsecast/control-auth.env + set +a make load - # or: API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js + # or: API_URL=https://api-pulse.noelsabia.dev \ + # CONTROL_USER="$CONTROL_USER" CONTROL_PASS="$CONTROL_PASS" \ + # ./k6/k6-sse run k6/sse.js ``` `make load` writes `k6/results/summary-.json` (git-ignored). -3. During the run, post a scenario change so the scenario-event latency has - samples and the propagation path is exercised: +3. The `presenter` scenario starts at `2m30s`, POSTs + `PRESENTER_SCENARIO=rainy-saturday` once to `/api/scenario`, and records the + request duration plus success. All 120 SSE clients must receive the + resulting `scenario` event. Override the timing/scenario if needed: ```bash - curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \ - -H 'Content-Type: application/json' \ - -d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \ - https://api-pulse.noelsabia.dev/api/scenario + make load PRESENTER_AT=3m PRESENTER_SCENARIO=sunny-saturday-market ``` - Repeat every 30–60 s with different scenarios if you want a distribution. - Without a change, `sse_event_latency` is measured from heartbeat `ping` - events, which still proves the stream is live. - 4. Record these numbers in the paper/README from the saved summary: - `sse_connections_opened` count (must be 120) - - `sse_connections_closed` count (must equal connections opened) + - `sse_connections_closed` count (must be >= 120) - `sse_open_success` rate (must be >= 99%) - `sse_event_latency` p95 (target < 2000 ms) + - `sse_scenario_latency` p95 (target < 2000 ms) + - `presenter_post_success` rate (must be 100%) + - `presenter_post_duration` (record p95/max) - `sse_connection_errors` (target 0) - `sse_events_received` total ## Local validation before the cluster run The script was validated locally on 2026-11-14 with the pinned k6 v2.0.0 build -against a mock SSE stream and the real API image (3 VUs, 1 heartbeat each): -connections opened and closed cleanly, open success 100%. The real 120-client -run still requires the deployed API, because localhost results say nothing -about Cloudflare buffering or SKE scheduling. +against a mock SSE stream and the real v1.1 API image (3 VUs, 1 heartbeat +each): connections opened and closed cleanly, open success 100%, presenter +POST 200, and scenario latency sampled. The real 120-client run still requires +the deployed API, because localhost results say nothing about Cloudflare +buffering or SKE scheduling. ## Pass criteria | Metric | Target | |---|---| | Connections opened | 120 | -| Connections closed (stream ended) | 120 | +| Connections closed (stream ended) | >= 120 | | Open success rate | >= 99% | | p95 event latency | < 2000 ms | +| p95 scenario latency | < 2000 ms | +| Presenter POST success | 100% | | Unexpected connection errors | 0 | | Duration | 5 minutes | -The k6 thresholds enforce connections opened, open success, and p95 latency; -the others are read from the summary. A final `sse_connections_closed` below -120 means at least one opened VU did not finish; inspect -`sse_connection_errors` for unexpected errors. A p95 above 2 s means Cloudflare, the Go SSE hub, or the -client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s -heartbeat, and whether the connection is HTTP/2-buffered at the edge. +The k6 thresholds enforce connections opened/closed, open success, p95 event +latency, p95 scenario latency, presenter success, and zero connection errors. +A p95 above 2 s means Cloudflare, the Go SSE hub, or the client is +buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s heartbeat, +and whether the connection is HTTP/2-buffered at the edge. + +## Results + +Populate from the saved `k6/results/summary-*.json`; never estimate. + +| Run | Date | Opened | Closed | Open success | p95 event | p95 scenario | Presenter success | Errors | +|---|---|---|---|---|---|---|---|---| +| v1.1 through Cloudflare | pending post-merge run | | | | | | | | ## Notes @@ -100,8 +117,9 @@ heartbeat, and whether the connection is HTTP/2-buffered at the edge. on NTP. Negative or > 1 h values are ignored. - The initial `scenario` event on connect replays the current state and carries the original state-change time, not the send time. The script excludes that - first scenario event per VU; only scenario changes broadcast during the run - are sampled by `sse_scenario_latency`. Heartbeat pings always count. + first scenario event per VU; the presenter POST is the scenario sample for + `sse_scenario_latency`. Heartbeat pings always count toward + `sse_event_latency`. - Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends cleanly instead of waiting for the 7-minute request timeout. - One API replica means all clients share one SSE hub; a pod restart drops diff --git a/k6/sse.js b/k6/sse.js index cfc5f90..c29065e 100644 --- a/k6/sse.js +++ b/k6/sse.js @@ -6,23 +6,31 @@ // stock k6 binary. Build the pinned binary once: // make k6-build // +// The `presenter` scenario starts at 2m30s (configurable) and POSTs one +// scenario change so scenario propagation through Cloudflare is measured. +// // Run: -// API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js +// source ~/.config/pulsecast/control-auth.env +// API_URL=https://api-pulse.noelsabia.dev CONTROL_USER="$CONTROL_USER" \ +// CONTROL_PASS="$CONTROL_PASS" ./k6/k6-sse run k6/sse.js // make load // // Recorded metrics: // sse_connections_opened counter, expected >= SSE_VUS (120) -// sse_connections_closed counter, expected to equal connections_opened +// sse_connections_closed counter, expected >= SSE_VUS // sse_open_success rate of VUs that opened a stream // sse_event_latency p95 server-timestamp-to-client latency, ms -// sse_scenario_latency same, scenario events only (if any occur) +// sse_scenario_latency p95 scenario-event latency, ms // sse_ping_latency same, heartbeat events only // sse_events_received total events +// presenter_post_success rate of the single scenario POST +// presenter_post_duration duration of the scenario POST, ms // // Each VU closes its stream after SSE_HOLD_PINGS heartbeats (20 * 15 s = 5 min). // That keeps 120 streams open for the full window and closes cleanly. import http from 'k6/http'; import sse from 'k6/x/sse'; +import encoding from 'k6/encoding'; import { check } from 'k6'; import { Counter, Rate, Trend } from 'k6/metrics'; @@ -31,6 +39,11 @@ const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`; const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120)); const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20)); const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m'; +const CONTROL_USER = __ENV.CONTROL_USER || ''; +const CONTROL_PASS = __ENV.CONTROL_PASS || ''; +const PRESENTER_SCENARIO = __ENV.PRESENTER_SCENARIO || 'rainy-saturday'; +const PRESENTER_AT = __ENV.PRESENTER_AT || '2m30s'; +const PRESENTER_TIMEOUT = __ENV.PRESENTER_TIMEOUT || '10s'; const connectionsOpened = new Counter('sse_connections_opened'); const connectionsClosed = new Counter('sse_connections_closed'); @@ -40,12 +53,18 @@ const eventsReceived = new Counter('sse_events_received'); const eventLatency = new Trend('sse_event_latency', true); const scenarioLatency = new Trend('sse_scenario_latency', true); const pingLatency = new Trend('sse_ping_latency', true); +const presenterSuccess = new Rate('presenter_post_success'); +const presenterLatency = new Trend('presenter_post_duration', true); const thresholds = { sse_open_success: ['rate>=0.99'], sse_event_latency: ['p(95)<2000'], + sse_scenario_latency: ['p(95)<2000'], + presenter_post_success: ['rate==1'], + sse_connection_errors: ['count==0'], }; thresholds[`sse_connections_opened`] = [`count>=${VUS}`]; +thresholds[`sse_connections_closed`] = [`count>=${VUS}`]; export const options = { scenarios: { @@ -56,6 +75,15 @@ export const options = { maxDuration: MAX_DURATION, gracefulStop: '30s', }, + presenter: { + executor: 'shared-iterations', + vus: 1, + iterations: 1, + startTime: PRESENTER_AT, + maxDuration: '30s', + gracefulStop: '5s', + exec: 'presenter', + }, }, thresholds, // Every VU owns one long-lived connection; do not pool them. @@ -71,6 +99,39 @@ export function setup() { return { api: API_URL, stream: STREAM_URL, vus: VUS }; } +// One presenter action during the run. This is deliberately separate from the +// SSE VUs so the POST does not block a client connection. +export function presenter() { + if (!CONTROL_USER || !CONTROL_PASS) { + throw new Error( + 'CONTROL_USER and CONTROL_PASS are required for the presenter scenario; source ~/.config/pulsecast/control-auth.env', + ); + } + + const payload = JSON.stringify({ + scenario: PRESENTER_SCENARIO, + weather: { temp_delta_c: 4, precip_mm: 0 }, + }); + const basic = encoding.b64encode(`${CONTROL_USER}:${CONTROL_PASS}`); + const res = http.post(`${API_URL}/api/scenario`, payload, { + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${basic}`, + }, + timeout: PRESENTER_TIMEOUT, + tags: { endpoint: 'presenter' }, + }); + + presenterLatency.add(res.timings.duration); + const ok = check(res, { + 'presenter scenario 200': (r) => r.status === 200, + }); + presenterSuccess.add(ok); + if (!ok) { + console.error(`presenter POST failed: status=${res.status} body=${res.body}`); + } +} + export default function () { let opened = false; let closedByUs = false; diff --git a/scripts/check_manifests.py b/scripts/check_manifests.py index 61314bf..e0144da 100755 --- a/scripts/check_manifests.py +++ b/scripts/check_manifests.py @@ -79,7 +79,7 @@ for name, cm in configmaps.items(): required_config = { "pulsecast-shared": {"PUBLIC_WEB_URL", "PUBLIC_API_URL", "API_INTERNAL_URL", "ML_INTERNAL_URL"}, "web-config": {"PORT", "NUXT_PUBLIC_API_BASE", "NUXT_API_INTERNAL_BASE"}, - "api-config": {"PORT", "DATA_DIR", "FASTAPI_URL", "ML_TIMEOUT", "SSE_HEARTBEAT"}, + "api-config": {"PORT", "FASTAPI_URL", "BOOTSTRAP_REFRESH", "ML_TIMEOUT", "SSE_HEARTBEAT"}, "ml-config": { "PORT", "PULSECAST_ROOT", "PULSECAST_ARTIFACTS", "PULSECAST_WEATHER", "PULSECAST_MODELS", "PULSECAST_UPLOADS", @@ -93,6 +93,8 @@ for name, keys in required_config.items(): have = set((cm.get("data") or {}).keys()) for key in sorted(keys - have): errors.append(f"ConfigMap/{name}: missing key {key}") +if "DATA_DIR" in ((configmaps.get("api-config") or {}).get("data") or {}): + errors.append("ConfigMap/api-config: DATA_DIR is no longer used by the v1.1 API") expected_ports = {"web": 3000, "api": 8080, "ml": 8000, "cloudflared": 2000} expected_uids = {"web": 10001, "api": 65532, "ml": 10001, "cloudflared": 65532} @@ -202,6 +204,11 @@ for name in ("web", "api", "ml", "cloudflared"): if ref not in configmaps: errors.append(f"Deployment/{name}: references missing ConfigMap/{ref}") expected_secret_refs = { + "web": { + ("control-auth", "CONTROL_USER"), + ("control-auth", "CONTROL_PASS"), + ("control-auth", "CONTROL_TOKEN"), + }, "api": { ("control-auth", "CONTROL_USER"), ("control-auth", "CONTROL_PASS"), diff --git a/scripts/gate1-check.sh b/scripts/gate1-check.sh index 2959db6..370d9f3 100755 --- a/scripts/gate1-check.sh +++ b/scripts/gate1-check.sh @@ -216,7 +216,7 @@ for deployment in web api ml cloudflared; do bad "Deployment/$deployment not ready (${ready:-0}/${desired:-?}, $image)" fi if [[ "$deployment" != "cloudflared" && "$image" != *":$TAG" ]]; then - warn "Deployment/$deployment image tag is not :$TAG" + bad "Deployment/$deployment image is ${image:-}, expected :$TAG" fi done -- 2.54.0