feat: prepare v1.1 deploy and presenter load test
infra-ci / manifests (push) Failing after 4s
infra-ci / manifests (pull_request) Failing after 3s

- build and push v1.1 images from the service main worktrees
- switch manifests to :v1.1; add web control Secret refs
- configure API BOOTSTRAP_REFRESH=60s and remove unused DATA_DIR
- add k6 presenter scenario with a mid-run control POST
- require control credentials in make load; update runbook and drills
This commit is contained in:
Noel Sabia committed 2026-11-14 08:00:58 +01:00
1 parent 40bc8e47d0
commit b98a1bab42
12 files changed
+264 -117

No files matched your search

+22 -10
View File
@@ -11,20 +11,20 @@ PYTHON ?= python3
REGISTRY ?= git.noelsabia.dev
OWNER ?= noel
TAG ?= gate1
TAG ?= v1.1
WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG)
# Service repos are bare Git repos beside this one, each with per-task worktrees:
# Hackathon_2026/web-nuxt.git/feat-web-bootstrap
# Hackathon_2026/api-go.git/feat-api-bootstrap
# Hackathon_2026/ML-py.git/feat-ml-bootstrap
# Release images are built from the service integration branches:
# Hackathon_2026/web-nuxt.git/main
# Hackathon_2026/api-go.git/main
# Hackathon_2026/ML-py.git/main
# Override any *_DIR on the command line when the layout differs.
SERVICE_REPOS ?= $(abspath $(ROOT)/../..)
WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/main)))
API_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/main)))
ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main)))
WEB_DIR ?= $(SERVICE_REPOS)/web-nuxt.git/main
API_DIR ?= $(SERVICE_REPOS)/api-go.git/main
ML_DIR ?= $(SERVICE_REPOS)/ML-py.git/main
WEB_URL ?= https://pulse.noelsabia.dev
API_URL ?= https://api-pulse.noelsabia.dev
@@ -36,6 +36,11 @@ SKIP_ML ?= 0
SSE_VUS ?= 120
SSE_HOLD_PINGS ?= 20
SSE_MAX_DURATION ?= 6m
CONTROL_USER ?=
CONTROL_PASS ?=
PRESENTER_SCENARIO ?= rainy-saturday
PRESENTER_AT ?= 2m30s
PRESENTER_TIMEOUT ?= 10s
K6 ?=
K6_BIN ?= $(ROOT)/k6/k6-sse
# xk6-sse v0.2.0 exposes client.on/client.close correctly on k6 v2.0.x but not
@@ -153,8 +158,13 @@ smoke-local: ## Smoke the three local dev servers
SMOKE_SSE="$(SMOKE_SSE)" SKIP_ML="$(SKIP_ML)" \
"$(ROOT)/scripts/smoke.sh"
load: ## Run the 120-client / 5-minute SSE load test
load: ## Run the 120-client / 5-minute SSE load test + one presenter POST
@mkdir -p "$(K6_RESULTS)"
@if [ -z "$(CONTROL_USER)" ] || [ -z "$(CONTROL_PASS)" ]; then \
echo "CONTROL_USER and CONTROL_PASS are required for the presenter scenario."; \
echo "Run: set -a; source ~/.config/pulsecast/control-auth.env; set +a"; \
exit 1; \
fi
@set -e; \
if [ -n "$(K6)" ]; then \
runner="$(K6)"; \
@@ -166,7 +176,9 @@ load: ## Run the 120-client / 5-minute SSE load test
echo "No k6 binary. Run 'make k6-build' once (needs Go), then retry."; \
exit 1; \
fi; \
export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \
export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" SSE_MAX_DURATION="$(SSE_MAX_DURATION)" \
CONTROL_USER="$(CONTROL_USER)" CONTROL_PASS="$(CONTROL_PASS)" \
PRESENTER_SCENARIO="$(PRESENTER_SCENARIO)" PRESENTER_AT="$(PRESENTER_AT)" PRESENTER_TIMEOUT="$(PRESENTER_TIMEOUT)"; \
stamp=$$(date +%Y%m%dT%H%M%S); \
echo "Using $$runner"; \
"$$runner" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"
+12 -8
View File
@@ -45,17 +45,21 @@ hostname, and verification steps.
## Gate 1 status
- Manifests validate read-only; `make gate1-check` is green except for pending
human prerequisites.
- All three images build and pass local manifest-equivalent runtime checks
- Static `make gate1-check STATIC_ONLY=1`/CI passes. The live check correctly
flags the cluster's v1.0 images until the reviewed v1.1 branch is applied.
- All three v1.1 images build and pass local manifest-equivalent runtime checks
(read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker.
- Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from
`git.noelsabia.dev/noel/hello:gate1`; the Service returned HTTP 200.
- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes;
`https://pulse.noelsabia.dev` returns 200 from hello with valid TLS. The API
hostname is `api-pulse.noelsabia.dev` because Universal SSL only covers
`*.noelsabia.dev`.
- Still needed: `control-auth` Secret, image push, deploy, and public smoke.
- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes.
`pulse` still points at the temporary hello Service until the v1.1 cutover.
The API hostname is `api-pulse.noelsabia.dev` because Universal SSL only
covers `*.noelsabia.dev`.
- v1.1 images are built from the service `main` worktrees, locally checked,
and pushed. The cluster still runs the previous v1.0 images until this branch
is reviewed/merged and explicitly applied.
- The v1.1 branch adds web control credentials, API bootstrap config, and the
k6 presenter scenario; load-test results are recorded after the approved run.
## CI
+3 -1
View File
@@ -42,8 +42,10 @@ metadata:
app.kubernetes.io/part-of: pulsecast
data:
PORT: "8080"
DATA_DIR: "/app/data"
FASTAPI_URL: "http://ml:8000"
# v1.1 fetches FastAPI /bootstrap at startup and every BOOTSTRAP_REFRESH;
# there is no baked data directory and no DATA_DIR env.
BOOTSTRAP_REFRESH: "60s"
ML_TIMEOUT: "1.5s"
SSE_HEARTBEAT: "15s"
---
+3 -3
View File
@@ -35,9 +35,9 @@ spec:
fsGroup: 10001
containers:
- name: ml
image: git.noelsabia.dev/noel/pulsecast-ml:gate1
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
# Switch to an immutable tag + IfNotPresent at deploy freeze.
image: git.noelsabia.dev/noel/pulsecast-ml:v1.1
# v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible.
# Switch back to IfNotPresent once the tag is frozen.
imagePullPolicy: Always
ports:
- name: http
+3 -3
View File
@@ -36,9 +36,9 @@ spec:
fsGroup: 65532
containers:
- name: api
image: git.noelsabia.dev/noel/pulsecast-api:gate1
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
# Switch to an immutable tag + IfNotPresent at deploy freeze.
image: git.noelsabia.dev/noel/pulsecast-api:v1.1
# v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible.
# Switch back to IfNotPresent once the tag is frozen.
imagePullPolicy: Always
ports:
- name: http
+20 -3
View File
@@ -39,9 +39,9 @@ spec:
containers:
- name: web
# Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`.
image: git.noelsabia.dev/noel/pulsecast-web:gate1
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
# Switch to an immutable tag + IfNotPresent at deploy freeze.
image: git.noelsabia.dev/noel/pulsecast-web:v1.1
# v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible.
# Switch back to IfNotPresent once the tag is frozen.
imagePullPolicy: Always
ports:
- name: http
@@ -58,6 +58,23 @@ spec:
# Nuxt/Nitro may use a temp dir; keep it on the writable emptyDir.
- name: NITRO_TEMP_DIR
value: /tmp
# The web server validates /control and POST /api/scenario itself
# and attaches these same credentials to Go control calls.
- name: CONTROL_USER
valueFrom:
secretKeyRef:
name: control-auth
key: CONTROL_USER
- name: CONTROL_PASS
valueFrom:
secretKeyRef:
name: control-auth
key: CONTROL_PASS
- name: CONTROL_TOKEN
valueFrom:
secretKeyRef:
name: control-auth
key: CONTROL_TOKEN
resources:
requests:
cpu: 100m
+75 -54
View File
@@ -24,7 +24,7 @@ Gate 1 passes only when **all** of these are true:
Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest
dry-run, or a container that only runs on your laptop.
## Current external status (2026-11-14 ~06:50 CET)
## Current external status (2026-11-14 ~08:00 CET)
Re-check before trusting this snapshot:
@@ -33,8 +33,8 @@ Re-check before trusting this snapshot:
Open-Meteo, and `git.noelsabia.dev`.
- **Tunnel active:** `pulsecast-gate1` has routes for `pulse.noelsabia.dev`
and `api-pulse.noelsabia.dev`; `cloudflared` is `1/1` with four HTTP/2
connections, and `https://pulse.noelsabia.dev` returns 200 from hello with
valid TLS. `api-pulse` returns 502 until the API is deployed.
connections. `api-pulse` serves the deployed API; `pulse` still points at
the temporary hello Service until the web cutover.
- **DNS works:** public resolvers return Cloudflare IPs for
`pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`. A local
`systemd-resolved` negative cache can be cleared with
@@ -42,20 +42,19 @@ Re-check before trusting this snapshot:
- **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only
`*.noelsabia.dev`; a two-level `api.pulse.noelsabia.dev` is not covered by
that certificate.
- **Gitea pull is proven (2026-11-14 ~06:16 CET):** `gitea-regcred` exists and
the cluster pulled `git.noelsabia.dev/noel/hello:gate1` in 120 ms; the hello
pod is `1/1 Running` and its Service returned HTTP 200. The Gitea owner is
`noel`.
- **Gitea pull is proven:** `gitea-regcred` exists; hello and the v1.0 service
images pulled from Gitea. The Gitea owner is `noel`.
- **v1.1 images pushed:** web/api/ml `:v1.1` were built from the service
`main` worktrees, locally exercised under numeric UIDs/read-only root, and
pushed. The cluster still runs the previous v1.0 images until the v1.1
branch is reviewed/merged and explicitly applied.
- **All three Secrets exist:** `gitea-regcred`, `cloudflared`, `control-auth`.
- **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the
`cloudflared` manifest uses `--protocol http2` instead of relying on
unverified UDP.
- **Numeric UIDs are required:** a non-numeric image user breaks
`runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in
`deploy/k8s/` sets explicit numeric `runAsUser`/`runAsGroup`/`fsGroup`.
- **Repo status (2026-11-14 ~06:50 CET):** all three images build and pass
local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp`
tmpfs). Remaining steps are the image push, `control-auth`, deploy, and
public smoke.
## What is possible now vs what is blocked
@@ -63,12 +62,10 @@ Re-check before trusting this snapshot:
|---|---|---|
| Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) |
| Gitea push login + hello-world push | human | **done**; hello pull and Service verified |
| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` + `cloudflared` **done**; `control-auth` pending |
| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active, public hello 200 |
| `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally |
| `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally |
| `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally |
| Full deploy + smoke + Gate 1 sign-off | human | **ready once the tunnel/secret/image steps are done** |
| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **done** |
| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active; `pulse` still points at hello |
| `api`, `ml`, `web` images | human | **v1.1 pushed**; local manifest-equivalent checks pass |
| v1.1 apply + smoke + presenter k6 + drills | human | **blocked on branch review/merge** |
`make build` and `make push` cover all three services; no Dockerfile is
missing now. Do not deploy placeholder images under the real service names.
@@ -86,7 +83,7 @@ export NS="fch26-onemanarmy"
export CTX="fch26-onemanarmy-sa-context"
export REGISTRY="git.noelsabia.dev"
export OWNER="noel"
export TAG="gate1"
export TAG="v1.1"
chmod 600 "$KUBECONFIG"
```
@@ -147,7 +144,7 @@ Run top to bottom. Stop at the first failure and use
- [ ] 5. `cloudflared` Secret created; logs show `Registered tunnel connection`.
- [ ] 6. Public hello route returns HTTP 200 (proves tunnel + DNS + TLS).
- [ ] 7. `control-auth` Secret created.
- [ ] 8. All three images built, tagged `gate1`, and pushed.
- [ ] 8. All three v1.1 images built and pushed.
- [ ] 9. `make validate` and `make gate1-check` pass.
- [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready.
- [ ] 11. `make smoke SMOKE_SSE=1` passes.
@@ -411,10 +408,10 @@ kubectl --context "$CTX" -n "$NS" describe pod -l app.kubernetes.io/name=cloudfl
## Step 4 — control-auth Secret
The API reads `CONTROL_USER` and `CONTROL_PASS` for browser control auth.
Both API and ML also need the same `CONTROL_TOKEN`: the API forwards it as a
Bearer token to FastAPI's internal `/train` routes. Generate both locally;
never commit them.
The web server validates `/control` and POST `/api/scenario` with
`CONTROL_USER`/`CONTROL_PASS`; web, API, and ML also need the same
`CONTROL_TOKEN` (the API forwards it as a Bearer token to FastAPI's internal
`/train` routes). Generate both locally; never commit them.
```bash
read -rsp 'Control password: ' CONTROL_PASS; echo
@@ -438,9 +435,9 @@ creates nothing.
Check whether the Dockerfiles exist:
```bash
test -f ../../web-nuxt.git/feat-web-bootstrap/Dockerfile && echo web-ok || echo web-blocked
test -f ../../api-go.git/feat-api-bootstrap/Dockerfile && echo api-ok || echo api-blocked
test -f ../../ML-py.git/feat-ml-bootstrap/Dockerfile && echo ml-ok || echo ml-blocked
test -f ../../web-nuxt.git/main/Dockerfile && echo web-ok || echo web-blocked
test -f ../../api-go.git/main/Dockerfile && echo api-ok || echo api-blocked
test -f ../../ML-py.git/main/Dockerfile && echo ml-ok || echo ml-blocked
```
Expected **now** (all three Dockerfiles landed on 2026-11-14):
@@ -457,18 +454,20 @@ Build all three:
make build
```
All three images were verified locally with the manifest's runtime
All three v1.1 images were verified locally with the manifest's runtime
constraints:
- API: distroless `static:nonroot`, `--user 65532:65532`, read-only root,
`/tmp` tmpfs; `/api/healthz` and `/api/readyz` returned 200.
- ML: image `USER app` is non-numeric, but the manifest overrides it with
`runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/healthz` and
`/readyz` returned 200 and `/meta` loaded the baked model.
- Web: image `USER node` is non-numeric, but the manifest overrides it with
`runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/` returned
HTTP 200 and the runtime `NUXT_PUBLIC_API_BASE` appeared in the HTML.
- No image was pushed or deployed by the infra agent.
`/tmp` tmpfs; `/api/healthz` and `/api/readyz` returned 200, and the API
loaded ML `/bootstrap` into memory.
- ML: image `USER 10001:10001`; with read-only root and `/tmp` tmpfs,
`/healthz` and `/readyz` returned 200, `/bootstrap` returned 8 sensors, and
`/meta` showed `mae=3.32`, `wape=27.5`, `r2=0.777`.
- Web: image `USER 10001:10001`; with read-only root and `/tmp` tmpfs, `/`,
`/stage`, and `/live` returned 200; `/control` returned 401 anonymous and
200 with control auth; the same-origin SSE proxy emitted events.
- v1.1 was pushed to Gitea. The cluster is not updated until this branch is
reviewed/merged and explicitly applied.
Push:
@@ -482,9 +481,9 @@ repeat the push login from Step 1.
The Makefile auto-detects the service worktrees:
```text
../../web-nuxt.git/feat-web-bootstrap
../../api-go.git/feat-api-bootstrap
../../ML-py.git/feat-ml-bootstrap
../../web-nuxt.git/main
../../api-go.git/main
../../ML-py.git/main
```
Override when the layout differs:
@@ -512,9 +511,9 @@ Each service image must:
| Service | Image | Port | UID:GID | Health paths |
|---|---|---|---|---|
| web | `git.noelsabia.dev/noel/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` |
| api | `git.noelsabia.dev/noel/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` |
| ml | `git.noelsabia.dev/noel/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` |
| web | `git.noelsabia.dev/noel/pulsecast-web:v1.1` | 3000 | `10001:10001` | `/`, `/stage`, `/live`, `/control` |
| api | `git.noelsabia.dev/noel/pulsecast-api:v1.1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` |
| ml | `git.noelsabia.dev/noel/pulsecast-ml:v1.1` | 8000 | `10001:10001` | `/healthz`, `/readyz`, `/bootstrap` |
The manifests set `runAsUser`/`runAsGroup`/`fsGroup` explicitly, so the image's
named user is irrelevant; the files must be readable by that numeric UID.
@@ -522,17 +521,19 @@ named user is irrelevant; the files must be readable by that numeric UID.
Dockerfile genuinely needs another UID, change the manifest and this table
together, and keep the value numeric.
`api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`,
`CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and
`ML_TIMEOUT`.
`web` must have `NUXT_PUBLIC_API_BASE=https://api-pulse.noelsabia.dev` baked in
or read it at runtime from the ConfigMap; the browser must use the public API
hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`,
`PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and
`CONTROL_TOKEN`; its image CMD binds `0.0.0.0:8000`. The env names in the
manifests are the integration point: change them here first if a service needs
a different name. The ML image currently uses a named `app` user; the manifest
overrides it with numeric UID 10001, which was verified locally.
`api` v1.1 reads `FASTAPI_URL=http://ml:8000`, `BOOTSTRAP_REFRESH=60s`,
`BOOTSTRAP_TIMEOUT`, `PORT`, `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`,
`SSE_HEARTBEAT`, and `ML_TIMEOUT`. It has **no baked data and no `DATA_DIR`**;
it fetches ML `/bootstrap` into memory and refreshes it.
`web` proxies Go and SSE server-side using `NUXT_API_INTERNAL_BASE=http://api:8080`;
it also needs `CONTROL_USER`, `CONTROL_PASS`, and `CONTROL_TOKEN` to protect
`/control` and POST `/api/scenario`. `NUXT_PUBLIC_API_BASE` and the public SSE
URL are display/future configuration; the browser calls the Nuxt origin.
`ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`, `PULSECAST_WEATHER`,
`PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and `CONTROL_TOKEN`; its image CMD
binds `0.0.0.0:8000`. The env names in the manifests are the integration
point: change them here first if a service needs a different name. All three
images use numeric UIDs: web/ml `10001:10001`, API `65532:65532`.
## Step 6 — prove public HTTPS with hello
@@ -623,7 +624,7 @@ Expected: `web`, `api`, `ml`, and `cloudflared` each `1/1`; Services `web`,
### Re-pushing the same tag
The manifests use `imagePullPolicy: Always` while the tag stays `gate1`. After
The manifests use `imagePullPolicy: Always` while the tag stays `v1.1`. After
rebuilding and pushing the same tag, force a re-pull:
```bash
@@ -680,6 +681,26 @@ The SSE output must start with `retry: 3000`, contain a `scenario` event
immediately, and show a `ping` within 20 seconds. If events are delayed
through Cloudflare, see the SSE row in Troubleshooting.
### v1.1 product checks
```bash
curl -fsS https://api-pulse.noelsabia.dev/api/sensors | jq 'length'
curl -fsS https://api-pulse.noelsabia.dev/api/meta \
| jq '{data_version,model_version,metrics,cache}'
curl -fsS -o /dev/null -w 'stage %{http_code}\n' https://pulse.noelsabia.dev/stage
curl -fsS -o /dev/null -w 'live %{http_code}\n' https://pulse.noelsabia.dev/live
curl -sS -o /dev/null -w 'control anon %{http_code}\n' https://pulse.noelsabia.dev/control
set -a; source ~/.config/pulsecast/control-auth.env; set +a
curl -sS -u "$CONTROL_USER:$CONTROL_PASS" -o /dev/null \
-w 'control auth %{http_code}\n' https://pulse.noelsabia.dev/control
curl -sS -o /dev/null -w 'plan %{time_total}s\n' \
'https://api-pulse.noelsabia.dev/api/staffing?sensor=hn-stadtgalerie-01&scenario=base&standard=standard'
```
Expected: **8** sensors; metrics `mae=3.32`, `wape=27.5`, `r2=0.777` with a
`cache` object; `/stage` and `/live` 200; `/control` 401 anonymous and 200
with control auth; plan under 2 seconds.
In-cluster connectivity (optional; **creates and removes one short-lived
pod**, so the human runs it):
@@ -738,7 +759,7 @@ you need to reproduce the exact deploy.
| `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. |
| `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. |
| Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. |
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. |
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS http://ml:8000/bootstrap` from inside the namespace; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | v1.1 has no baked data. It needs ML reachable at `FASTAPI_URL` and a successful `/bootstrap`; check `BOOTSTRAP_REFRESH` and the ML readiness. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. |
| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. |
| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. |
| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. |
+8 -3
View File
@@ -7,8 +7,9 @@ drill.
## 1. FastAPI down (cached fallback)
Goal: the Go API keeps serving forecasts from baked JSON with `"cached": true`
and `/api/readyz` stays 200. FastAPI being down must not fail API readiness.
Goal: the Go API keeps serving its last good in-memory `/bootstrap` snapshot
with `"cached": true` and `/api/readyz` stays 200. FastAPI being down must not
fail API readiness after the snapshot has loaded once.
```bash
# Baseline
@@ -26,6 +27,9 @@ curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01
# Recover
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=1
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/ml --timeout=180s
# The API refreshes within BOOTSTRAP_REFRESH=60s; confirm the cache is fresh again.
sleep 65
curl -sS https://api-pulse.noelsabia.dev/api/meta | jq '.cache'
```
Pass: no 5xx from `/api/forecast`, `cached` flips to `true` within the 1.5 s
@@ -65,7 +69,8 @@ the tunnel token means updating the Secret and restarting the Deployment.
Goal: a single API pod restart causes a short 502 window, not a stuck UI. SSE
clients reconnect with `retry: 3000`; scenario state is in memory and resets to
the default, which is documented.
the default. The restarted API re-fetches ML `/bootstrap` before `/api/readyz`
turns 200; keep ML up during this drill.
```bash
# Watch from two terminals:
+45 -27
View File
@@ -1,7 +1,9 @@
# k6 load test
`k6/sse.js` holds 120 concurrent Server-Sent Events connections for 5 minutes
and reports connection count plus p95 event latency.
and reports connection count plus p95 event latency. A separate `presenter`
scenario POSTs one scenario change at 2m30s so the test measures event
propagation through Cloudflare, not just heartbeat latency.
## Why a custom k6 binary
@@ -36,62 +38,77 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
You should see `retry: 3000` and a `scenario` event immediately, then a
`ping` every 15 s.
2. Run the load test:
2. Load the control credentials and run the load test. The presenter scenario
requires `CONTROL_USER` and `CONTROL_PASS`; `make load` refuses to start
without them.
```bash
set -a
source ~/.config/pulsecast/control-auth.env
set +a
make load
# or: API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
# or: API_URL=https://api-pulse.noelsabia.dev \
# CONTROL_USER="$CONTROL_USER" CONTROL_PASS="$CONTROL_PASS" \
# ./k6/k6-sse run k6/sse.js
```
`make load` writes `k6/results/summary-<timestamp>.json` (git-ignored).
3. During the run, post a scenario change so the scenario-event latency has
samples and the propagation path is exercised:
3. The `presenter` scenario starts at `2m30s`, POSTs
`PRESENTER_SCENARIO=rainy-saturday` once to `/api/scenario`, and records the
request duration plus success. All 120 SSE clients must receive the
resulting `scenario` event. Override the timing/scenario if needed:
```bash
curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \
-H 'Content-Type: application/json' \
-d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \
https://api-pulse.noelsabia.dev/api/scenario
make load PRESENTER_AT=3m PRESENTER_SCENARIO=sunny-saturday-market
```
Repeat every 30–60 s with different scenarios if you want a distribution.
Without a change, `sse_event_latency` is measured from heartbeat `ping`
events, which still proves the stream is live.
4. Record these numbers in the paper/README from the saved summary:
- `sse_connections_opened` count (must be 120)
- `sse_connections_closed` count (must equal connections opened)
- `sse_connections_closed` count (must be >= 120)
- `sse_open_success` rate (must be >= 99%)
- `sse_event_latency` p95 (target < 2000 ms)
- `sse_scenario_latency` p95 (target < 2000 ms)
- `presenter_post_success` rate (must be 100%)
- `presenter_post_duration` (record p95/max)
- `sse_connection_errors` (target 0)
- `sse_events_received` total
## Local validation before the cluster run
The script was validated locally on 2026-11-14 with the pinned k6 v2.0.0 build
against a mock SSE stream and the real API image (3 VUs, 1 heartbeat each):
connections opened and closed cleanly, open success 100%. The real 120-client
run still requires the deployed API, because localhost results say nothing
about Cloudflare buffering or SKE scheduling.
against a mock SSE stream and the real v1.1 API image (3 VUs, 1 heartbeat
each): connections opened and closed cleanly, open success 100%, presenter
POST 200, and scenario latency sampled. The real 120-client run still requires
the deployed API, because localhost results say nothing about Cloudflare
buffering or SKE scheduling.
## Pass criteria
| Metric | Target |
|---|---|
| Connections opened | 120 |
| Connections closed (stream ended) | 120 |
| Connections closed (stream ended) | >= 120 |
| Open success rate | >= 99% |
| p95 event latency | < 2000 ms |
| p95 scenario latency | < 2000 ms |
| Presenter POST success | 100% |
| Unexpected connection errors | 0 |
| Duration | 5 minutes |
The k6 thresholds enforce connections opened, open success, and p95 latency;
the others are read from the summary. A final `sse_connections_closed` below
120 means at least one opened VU did not finish; inspect
`sse_connection_errors` for unexpected errors. A p95 above 2 s means Cloudflare, the Go SSE hub, or the
client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s
heartbeat, and whether the connection is HTTP/2-buffered at the edge.
The k6 thresholds enforce connections opened/closed, open success, p95 event
latency, p95 scenario latency, presenter success, and zero connection errors.
A p95 above 2 s means Cloudflare, the Go SSE hub, or the client is
buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s heartbeat,
and whether the connection is HTTP/2-buffered at the edge.
## Results
Populate from the saved `k6/results/summary-*.json`; never estimate.
| Run | Date | Opened | Closed | Open success | p95 event | p95 scenario | Presenter success | Errors |
|---|---|---|---|---|---|---|---|---|
| v1.1 through Cloudflare | pending post-merge run | | | | | | | |
## Notes
@@ -100,8 +117,9 @@ heartbeat, and whether the connection is HTTP/2-buffered at the edge.
on NTP. Negative or > 1 h values are ignored.
- The initial `scenario` event on connect replays the current state and carries
the original state-change time, not the send time. The script excludes that
first scenario event per VU; only scenario changes broadcast during the run
are sampled by `sse_scenario_latency`. Heartbeat pings always count.
first scenario event per VU; the presenter POST is the scenario sample for
`sse_scenario_latency`. Heartbeat pings always count toward
`sse_event_latency`.
- Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends
cleanly instead of waiting for the 7-minute request timeout.
- One API replica means all clients share one SSE hub; a pod restart drops
+64 -3
View File
@@ -6,23 +6,31 @@
// stock k6 binary. Build the pinned binary once:
// make k6-build
//
// The `presenter` scenario starts at 2m30s (configurable) and POSTs one
// scenario change so scenario propagation through Cloudflare is measured.
//
// Run:
// API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
// source ~/.config/pulsecast/control-auth.env
// API_URL=https://api-pulse.noelsabia.dev CONTROL_USER="$CONTROL_USER" \
// CONTROL_PASS="$CONTROL_PASS" ./k6/k6-sse run k6/sse.js
// make load
//
// Recorded metrics:
// sse_connections_opened counter, expected >= SSE_VUS (120)
// sse_connections_closed counter, expected to equal connections_opened
// sse_connections_closed counter, expected >= SSE_VUS
// sse_open_success rate of VUs that opened a stream
// sse_event_latency p95 server-timestamp-to-client latency, ms
// sse_scenario_latency same, scenario events only (if any occur)
// sse_scenario_latency p95 scenario-event latency, ms
// sse_ping_latency same, heartbeat events only
// sse_events_received total events
// presenter_post_success rate of the single scenario POST
// presenter_post_duration duration of the scenario POST, ms
//
// Each VU closes its stream after SSE_HOLD_PINGS heartbeats (20 * 15 s = 5 min).
// That keeps 120 streams open for the full window and closes cleanly.
import http from 'k6/http';
import sse from 'k6/x/sse';
import encoding from 'k6/encoding';
import { check } from 'k6';
import { Counter, Rate, Trend } from 'k6/metrics';
@@ -31,6 +39,11 @@ const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`;
const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120));
const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20));
const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m';
const CONTROL_USER = __ENV.CONTROL_USER || '';
const CONTROL_PASS = __ENV.CONTROL_PASS || '';
const PRESENTER_SCENARIO = __ENV.PRESENTER_SCENARIO || 'rainy-saturday';
const PRESENTER_AT = __ENV.PRESENTER_AT || '2m30s';
const PRESENTER_TIMEOUT = __ENV.PRESENTER_TIMEOUT || '10s';
const connectionsOpened = new Counter('sse_connections_opened');
const connectionsClosed = new Counter('sse_connections_closed');
@@ -40,12 +53,18 @@ const eventsReceived = new Counter('sse_events_received');
const eventLatency = new Trend('sse_event_latency', true);
const scenarioLatency = new Trend('sse_scenario_latency', true);
const pingLatency = new Trend('sse_ping_latency', true);
const presenterSuccess = new Rate('presenter_post_success');
const presenterLatency = new Trend('presenter_post_duration', true);
const thresholds = {
sse_open_success: ['rate>=0.99'],
sse_event_latency: ['p(95)<2000'],
sse_scenario_latency: ['p(95)<2000'],
presenter_post_success: ['rate==1'],
sse_connection_errors: ['count==0'],
};
thresholds[`sse_connections_opened`] = [`count>=${VUS}`];
thresholds[`sse_connections_closed`] = [`count>=${VUS}`];
export const options = {
scenarios: {
@@ -56,6 +75,15 @@ export const options = {
maxDuration: MAX_DURATION,
gracefulStop: '30s',
},
presenter: {
executor: 'shared-iterations',
vus: 1,
iterations: 1,
startTime: PRESENTER_AT,
maxDuration: '30s',
gracefulStop: '5s',
exec: 'presenter',
},
},
thresholds,
// Every VU owns one long-lived connection; do not pool them.
@@ -71,6 +99,39 @@ export function setup() {
return { api: API_URL, stream: STREAM_URL, vus: VUS };
}
// One presenter action during the run. This is deliberately separate from the
// SSE VUs so the POST does not block a client connection.
export function presenter() {
if (!CONTROL_USER || !CONTROL_PASS) {
throw new Error(
'CONTROL_USER and CONTROL_PASS are required for the presenter scenario; source ~/.config/pulsecast/control-auth.env',
);
}
const payload = JSON.stringify({
scenario: PRESENTER_SCENARIO,
weather: { temp_delta_c: 4, precip_mm: 0 },
});
const basic = encoding.b64encode(`${CONTROL_USER}:${CONTROL_PASS}`);
const res = http.post(`${API_URL}/api/scenario`, payload, {
headers: {
'Content-Type': 'application/json',
Authorization: `Basic ${basic}`,
},
timeout: PRESENTER_TIMEOUT,
tags: { endpoint: 'presenter' },
});
presenterLatency.add(res.timings.duration);
const ok = check(res, {
'presenter scenario 200': (r) => r.status === 200,
});
presenterSuccess.add(ok);
if (!ok) {
console.error(`presenter POST failed: status=${res.status} body=${res.body}`);
}
}
export default function () {
let opened = false;
let closedByUs = false;
+8 -1
View File
@@ -79,7 +79,7 @@ for name, cm in configmaps.items():
required_config = {
"pulsecast-shared": {"PUBLIC_WEB_URL", "PUBLIC_API_URL", "API_INTERNAL_URL", "ML_INTERNAL_URL"},
"web-config": {"PORT", "NUXT_PUBLIC_API_BASE", "NUXT_API_INTERNAL_BASE"},
"api-config": {"PORT", "DATA_DIR", "FASTAPI_URL", "ML_TIMEOUT", "SSE_HEARTBEAT"},
"api-config": {"PORT", "FASTAPI_URL", "BOOTSTRAP_REFRESH", "ML_TIMEOUT", "SSE_HEARTBEAT"},
"ml-config": {
"PORT", "PULSECAST_ROOT", "PULSECAST_ARTIFACTS", "PULSECAST_WEATHER",
"PULSECAST_MODELS", "PULSECAST_UPLOADS",
@@ -93,6 +93,8 @@ for name, keys in required_config.items():
have = set((cm.get("data") or {}).keys())
for key in sorted(keys - have):
errors.append(f"ConfigMap/{name}: missing key {key}")
if "DATA_DIR" in ((configmaps.get("api-config") or {}).get("data") or {}):
errors.append("ConfigMap/api-config: DATA_DIR is no longer used by the v1.1 API")
expected_ports = {"web": 3000, "api": 8080, "ml": 8000, "cloudflared": 2000}
expected_uids = {"web": 10001, "api": 65532, "ml": 10001, "cloudflared": 65532}
@@ -202,6 +204,11 @@ for name in ("web", "api", "ml", "cloudflared"):
if ref not in configmaps:
errors.append(f"Deployment/{name}: references missing ConfigMap/{ref}")
expected_secret_refs = {
"web": {
("control-auth", "CONTROL_USER"),
("control-auth", "CONTROL_PASS"),
("control-auth", "CONTROL_TOKEN"),
},
"api": {
("control-auth", "CONTROL_USER"),
("control-auth", "CONTROL_PASS"),
+1 -1
View File
@@ -216,7 +216,7 @@ for deployment in web api ml cloudflared; do
bad "Deployment/$deployment not ready (${ready:-0}/${desired:-?}, $image)"
fi
if [[ "$deployment" != "cloudflared" && "$image" != *":$TAG" ]]; then
warn "Deployment/$deployment image tag is not :$TAG"
bad "Deployment/$deployment image is ${image:-<unknown>}, expected :$TAG"
fi
done