Merge pull request 'feat: prepare v1.1 deploy and presenter load test' (#2) from feat/infra-bootstrap into main
infra-ci / manifests (push) Failing after 3s
infra-ci / manifests (push) Failing after 3s
Reviewed-on: #2
This commit was merged in pull request #2.
This commit is contained in:
commit
662b773cf0
12 files changed
+264
-117
No files matched your search
@@ -11,20 +11,20 @@ PYTHON ?= python3
|
||||
|
||||
REGISTRY ?= git.noelsabia.dev
|
||||
OWNER ?= noel
|
||||
TAG ?= gate1
|
||||
TAG ?= v1.1
|
||||
WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
|
||||
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
|
||||
ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG)
|
||||
|
||||
# Service repos are bare Git repos beside this one, each with per-task worktrees:
|
||||
# Hackathon_2026/web-nuxt.git/feat-web-bootstrap
|
||||
# Hackathon_2026/api-go.git/feat-api-bootstrap
|
||||
# Hackathon_2026/ML-py.git/feat-ml-bootstrap
|
||||
# Release images are built from the service integration branches:
|
||||
# Hackathon_2026/web-nuxt.git/main
|
||||
# Hackathon_2026/api-go.git/main
|
||||
# Hackathon_2026/ML-py.git/main
|
||||
# Override any *_DIR on the command line when the layout differs.
|
||||
SERVICE_REPOS ?= $(abspath $(ROOT)/../..)
|
||||
WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/main)))
|
||||
API_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/main)))
|
||||
ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main)))
|
||||
WEB_DIR ?= $(SERVICE_REPOS)/web-nuxt.git/main
|
||||
API_DIR ?= $(SERVICE_REPOS)/api-go.git/main
|
||||
ML_DIR ?= $(SERVICE_REPOS)/ML-py.git/main
|
||||
|
||||
WEB_URL ?= https://pulse.noelsabia.dev
|
||||
API_URL ?= https://api-pulse.noelsabia.dev
|
||||
@@ -36,6 +36,11 @@ SKIP_ML ?= 0
|
||||
SSE_VUS ?= 120
|
||||
SSE_HOLD_PINGS ?= 20
|
||||
SSE_MAX_DURATION ?= 6m
|
||||
CONTROL_USER ?=
|
||||
CONTROL_PASS ?=
|
||||
PRESENTER_SCENARIO ?= rainy-saturday
|
||||
PRESENTER_AT ?= 2m30s
|
||||
PRESENTER_TIMEOUT ?= 10s
|
||||
K6 ?=
|
||||
K6_BIN ?= $(ROOT)/k6/k6-sse
|
||||
# xk6-sse v0.2.0 exposes client.on/client.close correctly on k6 v2.0.x but not
|
||||
@@ -153,8 +158,13 @@ smoke-local: ## Smoke the three local dev servers
|
||||
SMOKE_SSE="$(SMOKE_SSE)" SKIP_ML="$(SKIP_ML)" \
|
||||
"$(ROOT)/scripts/smoke.sh"
|
||||
|
||||
load: ## Run the 120-client / 5-minute SSE load test
|
||||
load: ## Run the 120-client / 5-minute SSE load test + one presenter POST
|
||||
@mkdir -p "$(K6_RESULTS)"
|
||||
@if [ -z "$(CONTROL_USER)" ] || [ -z "$(CONTROL_PASS)" ]; then \
|
||||
echo "CONTROL_USER and CONTROL_PASS are required for the presenter scenario."; \
|
||||
echo "Run: set -a; source ~/.config/pulsecast/control-auth.env; set +a"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@set -e; \
|
||||
if [ -n "$(K6)" ]; then \
|
||||
runner="$(K6)"; \
|
||||
@@ -166,7 +176,9 @@ load: ## Run the 120-client / 5-minute SSE load test
|
||||
echo "No k6 binary. Run 'make k6-build' once (needs Go), then retry."; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \
|
||||
export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" SSE_MAX_DURATION="$(SSE_MAX_DURATION)" \
|
||||
CONTROL_USER="$(CONTROL_USER)" CONTROL_PASS="$(CONTROL_PASS)" \
|
||||
PRESENTER_SCENARIO="$(PRESENTER_SCENARIO)" PRESENTER_AT="$(PRESENTER_AT)" PRESENTER_TIMEOUT="$(PRESENTER_TIMEOUT)"; \
|
||||
stamp=$$(date +%Y%m%dT%H%M%S); \
|
||||
echo "Using $$runner"; \
|
||||
"$$runner" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"
|
||||
|
||||
@@ -45,17 +45,21 @@ hostname, and verification steps.
|
||||
|
||||
## Gate 1 status
|
||||
|
||||
- Manifests validate read-only; `make gate1-check` is green except for pending
|
||||
human prerequisites.
|
||||
- All three images build and pass local manifest-equivalent runtime checks
|
||||
- Static `make gate1-check STATIC_ONLY=1`/CI passes. The live check correctly
|
||||
flags the cluster's v1.0 images until the reviewed v1.1 branch is applied.
|
||||
- All three v1.1 images build and pass local manifest-equivalent runtime checks
|
||||
(read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker.
|
||||
- Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from
|
||||
`git.noelsabia.dev/noel/hello:gate1`; the Service returned HTTP 200.
|
||||
- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes;
|
||||
`https://pulse.noelsabia.dev` returns 200 from hello with valid TLS. The API
|
||||
hostname is `api-pulse.noelsabia.dev` because Universal SSL only covers
|
||||
`*.noelsabia.dev`.
|
||||
- Still needed: `control-auth` Secret, image push, deploy, and public smoke.
|
||||
- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes.
|
||||
`pulse` still points at the temporary hello Service until the v1.1 cutover.
|
||||
The API hostname is `api-pulse.noelsabia.dev` because Universal SSL only
|
||||
covers `*.noelsabia.dev`.
|
||||
- v1.1 images are built from the service `main` worktrees, locally checked,
|
||||
and pushed. The cluster still runs the previous v1.0 images until this branch
|
||||
is reviewed/merged and explicitly applied.
|
||||
- The v1.1 branch adds web control credentials, API bootstrap config, and the
|
||||
k6 presenter scenario; load-test results are recorded after the approved run.
|
||||
|
||||
## CI
|
||||
|
||||
|
||||
@@ -42,8 +42,10 @@ metadata:
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
data:
|
||||
PORT: "8080"
|
||||
DATA_DIR: "/app/data"
|
||||
FASTAPI_URL: "http://ml:8000"
|
||||
# v1.1 fetches FastAPI /bootstrap at startup and every BOOTSTRAP_REFRESH;
|
||||
# there is no baked data directory and no DATA_DIR env.
|
||||
BOOTSTRAP_REFRESH: "60s"
|
||||
ML_TIMEOUT: "1.5s"
|
||||
SSE_HEARTBEAT: "15s"
|
||||
---
|
||||
|
||||
@@ -35,9 +35,9 @@ spec:
|
||||
fsGroup: 10001
|
||||
containers:
|
||||
- name: ml
|
||||
image: git.noelsabia.dev/noel/pulsecast-ml:gate1
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
image: git.noelsabia.dev/noel/pulsecast-ml:v1.1
|
||||
# v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible.
|
||||
# Switch back to IfNotPresent once the tag is frozen.
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- name: http
|
||||
|
||||
@@ -36,9 +36,9 @@ spec:
|
||||
fsGroup: 65532
|
||||
containers:
|
||||
- name: api
|
||||
image: git.noelsabia.dev/noel/pulsecast-api:gate1
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
image: git.noelsabia.dev/noel/pulsecast-api:v1.1
|
||||
# v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible.
|
||||
# Switch back to IfNotPresent once the tag is frozen.
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- name: http
|
||||
|
||||
+20
-3
@@ -39,9 +39,9 @@ spec:
|
||||
containers:
|
||||
- name: web
|
||||
# Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`.
|
||||
image: git.noelsabia.dev/noel/pulsecast-web:gate1
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
image: git.noelsabia.dev/noel/pulsecast-web:v1.1
|
||||
# v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible.
|
||||
# Switch back to IfNotPresent once the tag is frozen.
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- name: http
|
||||
@@ -58,6 +58,23 @@ spec:
|
||||
# Nuxt/Nitro may use a temp dir; keep it on the writable emptyDir.
|
||||
- name: NITRO_TEMP_DIR
|
||||
value: /tmp
|
||||
# The web server validates /control and POST /api/scenario itself
|
||||
# and attaches these same credentials to Go control calls.
|
||||
- name: CONTROL_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: control-auth
|
||||
key: CONTROL_USER
|
||||
- name: CONTROL_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: control-auth
|
||||
key: CONTROL_PASS
|
||||
- name: CONTROL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: control-auth
|
||||
key: CONTROL_TOKEN
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
|
||||
+75
-54
@@ -24,7 +24,7 @@ Gate 1 passes only when **all** of these are true:
|
||||
Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest
|
||||
dry-run, or a container that only runs on your laptop.
|
||||
|
||||
## Current external status (2026-11-14 ~06:50 CET)
|
||||
## Current external status (2026-11-14 ~08:00 CET)
|
||||
|
||||
Re-check before trusting this snapshot:
|
||||
|
||||
@@ -33,8 +33,8 @@ Re-check before trusting this snapshot:
|
||||
Open-Meteo, and `git.noelsabia.dev`.
|
||||
- **Tunnel active:** `pulsecast-gate1` has routes for `pulse.noelsabia.dev`
|
||||
and `api-pulse.noelsabia.dev`; `cloudflared` is `1/1` with four HTTP/2
|
||||
connections, and `https://pulse.noelsabia.dev` returns 200 from hello with
|
||||
valid TLS. `api-pulse` returns 502 until the API is deployed.
|
||||
connections. `api-pulse` serves the deployed API; `pulse` still points at
|
||||
the temporary hello Service until the web cutover.
|
||||
- **DNS works:** public resolvers return Cloudflare IPs for
|
||||
`pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`. A local
|
||||
`systemd-resolved` negative cache can be cleared with
|
||||
@@ -42,20 +42,19 @@ Re-check before trusting this snapshot:
|
||||
- **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only
|
||||
`*.noelsabia.dev`; a two-level `api.pulse.noelsabia.dev` is not covered by
|
||||
that certificate.
|
||||
- **Gitea pull is proven (2026-11-14 ~06:16 CET):** `gitea-regcred` exists and
|
||||
the cluster pulled `git.noelsabia.dev/noel/hello:gate1` in 120 ms; the hello
|
||||
pod is `1/1 Running` and its Service returned HTTP 200. The Gitea owner is
|
||||
`noel`.
|
||||
- **Gitea pull is proven:** `gitea-regcred` exists; hello and the v1.0 service
|
||||
images pulled from Gitea. The Gitea owner is `noel`.
|
||||
- **v1.1 images pushed:** web/api/ml `:v1.1` were built from the service
|
||||
`main` worktrees, locally exercised under numeric UIDs/read-only root, and
|
||||
pushed. The cluster still runs the previous v1.0 images until the v1.1
|
||||
branch is reviewed/merged and explicitly applied.
|
||||
- **All three Secrets exist:** `gitea-regcred`, `cloudflared`, `control-auth`.
|
||||
- **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the
|
||||
`cloudflared` manifest uses `--protocol http2` instead of relying on
|
||||
unverified UDP.
|
||||
- **Numeric UIDs are required:** a non-numeric image user breaks
|
||||
`runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in
|
||||
`deploy/k8s/` sets explicit numeric `runAsUser`/`runAsGroup`/`fsGroup`.
|
||||
- **Repo status (2026-11-14 ~06:50 CET):** all three images build and pass
|
||||
local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp`
|
||||
tmpfs). Remaining steps are the image push, `control-auth`, deploy, and
|
||||
public smoke.
|
||||
|
||||
## What is possible now vs what is blocked
|
||||
|
||||
@@ -63,12 +62,10 @@ Re-check before trusting this snapshot:
|
||||
|---|---|---|
|
||||
| Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) |
|
||||
| Gitea push login + hello-world push | human | **done**; hello pull and Service verified |
|
||||
| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` + `cloudflared` **done**; `control-auth` pending |
|
||||
| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active, public hello 200 |
|
||||
| `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally |
|
||||
| `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally |
|
||||
| `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally |
|
||||
| Full deploy + smoke + Gate 1 sign-off | human | **ready once the tunnel/secret/image steps are done** |
|
||||
| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **done** |
|
||||
| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active; `pulse` still points at hello |
|
||||
| `api`, `ml`, `web` images | human | **v1.1 pushed**; local manifest-equivalent checks pass |
|
||||
| v1.1 apply + smoke + presenter k6 + drills | human | **blocked on branch review/merge** |
|
||||
|
||||
`make build` and `make push` cover all three services; no Dockerfile is
|
||||
missing now. Do not deploy placeholder images under the real service names.
|
||||
@@ -86,7 +83,7 @@ export NS="fch26-onemanarmy"
|
||||
export CTX="fch26-onemanarmy-sa-context"
|
||||
export REGISTRY="git.noelsabia.dev"
|
||||
export OWNER="noel"
|
||||
export TAG="gate1"
|
||||
export TAG="v1.1"
|
||||
|
||||
chmod 600 "$KUBECONFIG"
|
||||
```
|
||||
@@ -147,7 +144,7 @@ Run top to bottom. Stop at the first failure and use
|
||||
- [ ] 5. `cloudflared` Secret created; logs show `Registered tunnel connection`.
|
||||
- [ ] 6. Public hello route returns HTTP 200 (proves tunnel + DNS + TLS).
|
||||
- [ ] 7. `control-auth` Secret created.
|
||||
- [ ] 8. All three images built, tagged `gate1`, and pushed.
|
||||
- [ ] 8. All three v1.1 images built and pushed.
|
||||
- [ ] 9. `make validate` and `make gate1-check` pass.
|
||||
- [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready.
|
||||
- [ ] 11. `make smoke SMOKE_SSE=1` passes.
|
||||
@@ -411,10 +408,10 @@ kubectl --context "$CTX" -n "$NS" describe pod -l app.kubernetes.io/name=cloudfl
|
||||
|
||||
## Step 4 — control-auth Secret
|
||||
|
||||
The API reads `CONTROL_USER` and `CONTROL_PASS` for browser control auth.
|
||||
Both API and ML also need the same `CONTROL_TOKEN`: the API forwards it as a
|
||||
Bearer token to FastAPI's internal `/train` routes. Generate both locally;
|
||||
never commit them.
|
||||
The web server validates `/control` and POST `/api/scenario` with
|
||||
`CONTROL_USER`/`CONTROL_PASS`; web, API, and ML also need the same
|
||||
`CONTROL_TOKEN` (the API forwards it as a Bearer token to FastAPI's internal
|
||||
`/train` routes). Generate both locally; never commit them.
|
||||
|
||||
```bash
|
||||
read -rsp 'Control password: ' CONTROL_PASS; echo
|
||||
@@ -438,9 +435,9 @@ creates nothing.
|
||||
Check whether the Dockerfiles exist:
|
||||
|
||||
```bash
|
||||
test -f ../../web-nuxt.git/feat-web-bootstrap/Dockerfile && echo web-ok || echo web-blocked
|
||||
test -f ../../api-go.git/feat-api-bootstrap/Dockerfile && echo api-ok || echo api-blocked
|
||||
test -f ../../ML-py.git/feat-ml-bootstrap/Dockerfile && echo ml-ok || echo ml-blocked
|
||||
test -f ../../web-nuxt.git/main/Dockerfile && echo web-ok || echo web-blocked
|
||||
test -f ../../api-go.git/main/Dockerfile && echo api-ok || echo api-blocked
|
||||
test -f ../../ML-py.git/main/Dockerfile && echo ml-ok || echo ml-blocked
|
||||
```
|
||||
|
||||
Expected **now** (all three Dockerfiles landed on 2026-11-14):
|
||||
@@ -457,18 +454,20 @@ Build all three:
|
||||
make build
|
||||
```
|
||||
|
||||
All three images were verified locally with the manifest's runtime
|
||||
All three v1.1 images were verified locally with the manifest's runtime
|
||||
constraints:
|
||||
|
||||
- API: distroless `static:nonroot`, `--user 65532:65532`, read-only root,
|
||||
`/tmp` tmpfs; `/api/healthz` and `/api/readyz` returned 200.
|
||||
- ML: image `USER app` is non-numeric, but the manifest overrides it with
|
||||
`runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/healthz` and
|
||||
`/readyz` returned 200 and `/meta` loaded the baked model.
|
||||
- Web: image `USER node` is non-numeric, but the manifest overrides it with
|
||||
`runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/` returned
|
||||
HTTP 200 and the runtime `NUXT_PUBLIC_API_BASE` appeared in the HTML.
|
||||
- No image was pushed or deployed by the infra agent.
|
||||
`/tmp` tmpfs; `/api/healthz` and `/api/readyz` returned 200, and the API
|
||||
loaded ML `/bootstrap` into memory.
|
||||
- ML: image `USER 10001:10001`; with read-only root and `/tmp` tmpfs,
|
||||
`/healthz` and `/readyz` returned 200, `/bootstrap` returned 8 sensors, and
|
||||
`/meta` showed `mae=3.32`, `wape=27.5`, `r2=0.777`.
|
||||
- Web: image `USER 10001:10001`; with read-only root and `/tmp` tmpfs, `/`,
|
||||
`/stage`, and `/live` returned 200; `/control` returned 401 anonymous and
|
||||
200 with control auth; the same-origin SSE proxy emitted events.
|
||||
- v1.1 was pushed to Gitea. The cluster is not updated until this branch is
|
||||
reviewed/merged and explicitly applied.
|
||||
|
||||
Push:
|
||||
|
||||
@@ -482,9 +481,9 @@ repeat the push login from Step 1.
|
||||
The Makefile auto-detects the service worktrees:
|
||||
|
||||
```text
|
||||
../../web-nuxt.git/feat-web-bootstrap
|
||||
../../api-go.git/feat-api-bootstrap
|
||||
../../ML-py.git/feat-ml-bootstrap
|
||||
../../web-nuxt.git/main
|
||||
../../api-go.git/main
|
||||
../../ML-py.git/main
|
||||
```
|
||||
|
||||
Override when the layout differs:
|
||||
@@ -512,9 +511,9 @@ Each service image must:
|
||||
|
||||
| Service | Image | Port | UID:GID | Health paths |
|
||||
|---|---|---|---|---|
|
||||
| web | `git.noelsabia.dev/noel/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` |
|
||||
| api | `git.noelsabia.dev/noel/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` |
|
||||
| ml | `git.noelsabia.dev/noel/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` |
|
||||
| web | `git.noelsabia.dev/noel/pulsecast-web:v1.1` | 3000 | `10001:10001` | `/`, `/stage`, `/live`, `/control` |
|
||||
| api | `git.noelsabia.dev/noel/pulsecast-api:v1.1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` |
|
||||
| ml | `git.noelsabia.dev/noel/pulsecast-ml:v1.1` | 8000 | `10001:10001` | `/healthz`, `/readyz`, `/bootstrap` |
|
||||
|
||||
The manifests set `runAsUser`/`runAsGroup`/`fsGroup` explicitly, so the image's
|
||||
named user is irrelevant; the files must be readable by that numeric UID.
|
||||
@@ -522,17 +521,19 @@ named user is irrelevant; the files must be readable by that numeric UID.
|
||||
Dockerfile genuinely needs another UID, change the manifest and this table
|
||||
together, and keep the value numeric.
|
||||
|
||||
`api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`,
|
||||
`CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and
|
||||
`ML_TIMEOUT`.
|
||||
`web` must have `NUXT_PUBLIC_API_BASE=https://api-pulse.noelsabia.dev` baked in
|
||||
or read it at runtime from the ConfigMap; the browser must use the public API
|
||||
hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`,
|
||||
`PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and
|
||||
`CONTROL_TOKEN`; its image CMD binds `0.0.0.0:8000`. The env names in the
|
||||
manifests are the integration point: change them here first if a service needs
|
||||
a different name. The ML image currently uses a named `app` user; the manifest
|
||||
overrides it with numeric UID 10001, which was verified locally.
|
||||
`api` v1.1 reads `FASTAPI_URL=http://ml:8000`, `BOOTSTRAP_REFRESH=60s`,
|
||||
`BOOTSTRAP_TIMEOUT`, `PORT`, `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`,
|
||||
`SSE_HEARTBEAT`, and `ML_TIMEOUT`. It has **no baked data and no `DATA_DIR`**;
|
||||
it fetches ML `/bootstrap` into memory and refreshes it.
|
||||
`web` proxies Go and SSE server-side using `NUXT_API_INTERNAL_BASE=http://api:8080`;
|
||||
it also needs `CONTROL_USER`, `CONTROL_PASS`, and `CONTROL_TOKEN` to protect
|
||||
`/control` and POST `/api/scenario`. `NUXT_PUBLIC_API_BASE` and the public SSE
|
||||
URL are display/future configuration; the browser calls the Nuxt origin.
|
||||
`ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`, `PULSECAST_WEATHER`,
|
||||
`PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and `CONTROL_TOKEN`; its image CMD
|
||||
binds `0.0.0.0:8000`. The env names in the manifests are the integration
|
||||
point: change them here first if a service needs a different name. All three
|
||||
images use numeric UIDs: web/ml `10001:10001`, API `65532:65532`.
|
||||
|
||||
## Step 6 — prove public HTTPS with hello
|
||||
|
||||
@@ -623,7 +624,7 @@ Expected: `web`, `api`, `ml`, and `cloudflared` each `1/1`; Services `web`,
|
||||
|
||||
### Re-pushing the same tag
|
||||
|
||||
The manifests use `imagePullPolicy: Always` while the tag stays `gate1`. After
|
||||
The manifests use `imagePullPolicy: Always` while the tag stays `v1.1`. After
|
||||
rebuilding and pushing the same tag, force a re-pull:
|
||||
|
||||
```bash
|
||||
@@ -680,6 +681,26 @@ The SSE output must start with `retry: 3000`, contain a `scenario` event
|
||||
immediately, and show a `ping` within 20 seconds. If events are delayed
|
||||
through Cloudflare, see the SSE row in Troubleshooting.
|
||||
|
||||
### v1.1 product checks
|
||||
|
||||
```bash
|
||||
curl -fsS https://api-pulse.noelsabia.dev/api/sensors | jq 'length'
|
||||
curl -fsS https://api-pulse.noelsabia.dev/api/meta \
|
||||
| jq '{data_version,model_version,metrics,cache}'
|
||||
curl -fsS -o /dev/null -w 'stage %{http_code}\n' https://pulse.noelsabia.dev/stage
|
||||
curl -fsS -o /dev/null -w 'live %{http_code}\n' https://pulse.noelsabia.dev/live
|
||||
curl -sS -o /dev/null -w 'control anon %{http_code}\n' https://pulse.noelsabia.dev/control
|
||||
set -a; source ~/.config/pulsecast/control-auth.env; set +a
|
||||
curl -sS -u "$CONTROL_USER:$CONTROL_PASS" -o /dev/null \
|
||||
-w 'control auth %{http_code}\n' https://pulse.noelsabia.dev/control
|
||||
curl -sS -o /dev/null -w 'plan %{time_total}s\n' \
|
||||
'https://api-pulse.noelsabia.dev/api/staffing?sensor=hn-stadtgalerie-01&scenario=base&standard=standard'
|
||||
```
|
||||
|
||||
Expected: **8** sensors; metrics `mae=3.32`, `wape=27.5`, `r2=0.777` with a
|
||||
`cache` object; `/stage` and `/live` 200; `/control` 401 anonymous and 200
|
||||
with control auth; plan under 2 seconds.
|
||||
|
||||
In-cluster connectivity (optional; **creates and removes one short-lived
|
||||
pod**, so the human runs it):
|
||||
|
||||
@@ -738,7 +759,7 @@ you need to reproduce the exact deploy.
|
||||
| `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. |
|
||||
| `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. |
|
||||
| Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. |
|
||||
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. |
|
||||
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS http://ml:8000/bootstrap` from inside the namespace; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | v1.1 has no baked data. It needs ML reachable at `FASTAPI_URL` and a successful `/bootstrap`; check `BOOTSTRAP_REFRESH` and the ML readiness. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. |
|
||||
| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. |
|
||||
| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. |
|
||||
| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. |
|
||||
|
||||
@@ -7,8 +7,9 @@ drill.
|
||||
|
||||
## 1. FastAPI down (cached fallback)
|
||||
|
||||
Goal: the Go API keeps serving forecasts from baked JSON with `"cached": true`
|
||||
and `/api/readyz` stays 200. FastAPI being down must not fail API readiness.
|
||||
Goal: the Go API keeps serving its last good in-memory `/bootstrap` snapshot
|
||||
with `"cached": true` and `/api/readyz` stays 200. FastAPI being down must not
|
||||
fail API readiness after the snapshot has loaded once.
|
||||
|
||||
```bash
|
||||
# Baseline
|
||||
@@ -26,6 +27,9 @@ curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01
|
||||
# Recover
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=1
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/ml --timeout=180s
|
||||
# The API refreshes within BOOTSTRAP_REFRESH=60s; confirm the cache is fresh again.
|
||||
sleep 65
|
||||
curl -sS https://api-pulse.noelsabia.dev/api/meta | jq '.cache'
|
||||
```
|
||||
|
||||
Pass: no 5xx from `/api/forecast`, `cached` flips to `true` within the 1.5 s
|
||||
@@ -65,7 +69,8 @@ the tunnel token means updating the Secret and restarting the Deployment.
|
||||
|
||||
Goal: a single API pod restart causes a short 502 window, not a stuck UI. SSE
|
||||
clients reconnect with `retry: 3000`; scenario state is in memory and resets to
|
||||
the default, which is documented.
|
||||
the default. The restarted API re-fetches ML `/bootstrap` before `/api/readyz`
|
||||
turns 200; keep ML up during this drill.
|
||||
|
||||
```bash
|
||||
# Watch from two terminals:
|
||||
|
||||
+45
-27
@@ -1,7 +1,9 @@
|
||||
# k6 load test
|
||||
|
||||
`k6/sse.js` holds 120 concurrent Server-Sent Events connections for 5 minutes
|
||||
and reports connection count plus p95 event latency.
|
||||
and reports connection count plus p95 event latency. A separate `presenter`
|
||||
scenario POSTs one scenario change at 2m30s so the test measures event
|
||||
propagation through Cloudflare, not just heartbeat latency.
|
||||
|
||||
## Why a custom k6 binary
|
||||
|
||||
@@ -36,62 +38,77 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
|
||||
You should see `retry: 3000` and a `scenario` event immediately, then a
|
||||
`ping` every 15 s.
|
||||
|
||||
2. Run the load test:
|
||||
2. Load the control credentials and run the load test. The presenter scenario
|
||||
requires `CONTROL_USER` and `CONTROL_PASS`; `make load` refuses to start
|
||||
without them.
|
||||
|
||||
```bash
|
||||
set -a
|
||||
source ~/.config/pulsecast/control-auth.env
|
||||
set +a
|
||||
make load
|
||||
# or: API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
|
||||
# or: API_URL=https://api-pulse.noelsabia.dev \
|
||||
# CONTROL_USER="$CONTROL_USER" CONTROL_PASS="$CONTROL_PASS" \
|
||||
# ./k6/k6-sse run k6/sse.js
|
||||
```
|
||||
|
||||
`make load` writes `k6/results/summary-<timestamp>.json` (git-ignored).
|
||||
|
||||
3. During the run, post a scenario change so the scenario-event latency has
|
||||
samples and the propagation path is exercised:
|
||||
3. The `presenter` scenario starts at `2m30s`, POSTs
|
||||
`PRESENTER_SCENARIO=rainy-saturday` once to `/api/scenario`, and records the
|
||||
request duration plus success. All 120 SSE clients must receive the
|
||||
resulting `scenario` event. Override the timing/scenario if needed:
|
||||
|
||||
```bash
|
||||
curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \
|
||||
https://api-pulse.noelsabia.dev/api/scenario
|
||||
make load PRESENTER_AT=3m PRESENTER_SCENARIO=sunny-saturday-market
|
||||
```
|
||||
|
||||
Repeat every 30–60 s with different scenarios if you want a distribution.
|
||||
Without a change, `sse_event_latency` is measured from heartbeat `ping`
|
||||
events, which still proves the stream is live.
|
||||
|
||||
4. Record these numbers in the paper/README from the saved summary:
|
||||
- `sse_connections_opened` count (must be 120)
|
||||
- `sse_connections_closed` count (must equal connections opened)
|
||||
- `sse_connections_closed` count (must be >= 120)
|
||||
- `sse_open_success` rate (must be >= 99%)
|
||||
- `sse_event_latency` p95 (target < 2000 ms)
|
||||
- `sse_scenario_latency` p95 (target < 2000 ms)
|
||||
- `presenter_post_success` rate (must be 100%)
|
||||
- `presenter_post_duration` (record p95/max)
|
||||
- `sse_connection_errors` (target 0)
|
||||
- `sse_events_received` total
|
||||
|
||||
## Local validation before the cluster run
|
||||
|
||||
The script was validated locally on 2026-11-14 with the pinned k6 v2.0.0 build
|
||||
against a mock SSE stream and the real API image (3 VUs, 1 heartbeat each):
|
||||
connections opened and closed cleanly, open success 100%. The real 120-client
|
||||
run still requires the deployed API, because localhost results say nothing
|
||||
about Cloudflare buffering or SKE scheduling.
|
||||
against a mock SSE stream and the real v1.1 API image (3 VUs, 1 heartbeat
|
||||
each): connections opened and closed cleanly, open success 100%, presenter
|
||||
POST 200, and scenario latency sampled. The real 120-client run still requires
|
||||
the deployed API, because localhost results say nothing about Cloudflare
|
||||
buffering or SKE scheduling.
|
||||
|
||||
## Pass criteria
|
||||
|
||||
| Metric | Target |
|
||||
|---|---|
|
||||
| Connections opened | 120 |
|
||||
| Connections closed (stream ended) | 120 |
|
||||
| Connections closed (stream ended) | >= 120 |
|
||||
| Open success rate | >= 99% |
|
||||
| p95 event latency | < 2000 ms |
|
||||
| p95 scenario latency | < 2000 ms |
|
||||
| Presenter POST success | 100% |
|
||||
| Unexpected connection errors | 0 |
|
||||
| Duration | 5 minutes |
|
||||
|
||||
The k6 thresholds enforce connections opened, open success, and p95 latency;
|
||||
the others are read from the summary. A final `sse_connections_closed` below
|
||||
120 means at least one opened VU did not finish; inspect
|
||||
`sse_connection_errors` for unexpected errors. A p95 above 2 s means Cloudflare, the Go SSE hub, or the
|
||||
client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s
|
||||
heartbeat, and whether the connection is HTTP/2-buffered at the edge.
|
||||
The k6 thresholds enforce connections opened/closed, open success, p95 event
|
||||
latency, p95 scenario latency, presenter success, and zero connection errors.
|
||||
A p95 above 2 s means Cloudflare, the Go SSE hub, or the client is
|
||||
buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s heartbeat,
|
||||
and whether the connection is HTTP/2-buffered at the edge.
|
||||
|
||||
## Results
|
||||
|
||||
Populate from the saved `k6/results/summary-*.json`; never estimate.
|
||||
|
||||
| Run | Date | Opened | Closed | Open success | p95 event | p95 scenario | Presenter success | Errors |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| v1.1 through Cloudflare | pending post-merge run | | | | | | | |
|
||||
|
||||
## Notes
|
||||
|
||||
@@ -100,8 +117,9 @@ heartbeat, and whether the connection is HTTP/2-buffered at the edge.
|
||||
on NTP. Negative or > 1 h values are ignored.
|
||||
- The initial `scenario` event on connect replays the current state and carries
|
||||
the original state-change time, not the send time. The script excludes that
|
||||
first scenario event per VU; only scenario changes broadcast during the run
|
||||
are sampled by `sse_scenario_latency`. Heartbeat pings always count.
|
||||
first scenario event per VU; the presenter POST is the scenario sample for
|
||||
`sse_scenario_latency`. Heartbeat pings always count toward
|
||||
`sse_event_latency`.
|
||||
- Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends
|
||||
cleanly instead of waiting for the 7-minute request timeout.
|
||||
- One API replica means all clients share one SSE hub; a pod restart drops
|
||||
|
||||
@@ -6,23 +6,31 @@
|
||||
// stock k6 binary. Build the pinned binary once:
|
||||
// make k6-build
|
||||
//
|
||||
// The `presenter` scenario starts at 2m30s (configurable) and POSTs one
|
||||
// scenario change so scenario propagation through Cloudflare is measured.
|
||||
//
|
||||
// Run:
|
||||
// API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
|
||||
// source ~/.config/pulsecast/control-auth.env
|
||||
// API_URL=https://api-pulse.noelsabia.dev CONTROL_USER="$CONTROL_USER" \
|
||||
// CONTROL_PASS="$CONTROL_PASS" ./k6/k6-sse run k6/sse.js
|
||||
// make load
|
||||
//
|
||||
// Recorded metrics:
|
||||
// sse_connections_opened counter, expected >= SSE_VUS (120)
|
||||
// sse_connections_closed counter, expected to equal connections_opened
|
||||
// sse_connections_closed counter, expected >= SSE_VUS
|
||||
// sse_open_success rate of VUs that opened a stream
|
||||
// sse_event_latency p95 server-timestamp-to-client latency, ms
|
||||
// sse_scenario_latency same, scenario events only (if any occur)
|
||||
// sse_scenario_latency p95 scenario-event latency, ms
|
||||
// sse_ping_latency same, heartbeat events only
|
||||
// sse_events_received total events
|
||||
// presenter_post_success rate of the single scenario POST
|
||||
// presenter_post_duration duration of the scenario POST, ms
|
||||
//
|
||||
// Each VU closes its stream after SSE_HOLD_PINGS heartbeats (20 * 15 s = 5 min).
|
||||
// That keeps 120 streams open for the full window and closes cleanly.
|
||||
import http from 'k6/http';
|
||||
import sse from 'k6/x/sse';
|
||||
import encoding from 'k6/encoding';
|
||||
import { check } from 'k6';
|
||||
import { Counter, Rate, Trend } from 'k6/metrics';
|
||||
|
||||
@@ -31,6 +39,11 @@ const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`;
|
||||
const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120));
|
||||
const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20));
|
||||
const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m';
|
||||
const CONTROL_USER = __ENV.CONTROL_USER || '';
|
||||
const CONTROL_PASS = __ENV.CONTROL_PASS || '';
|
||||
const PRESENTER_SCENARIO = __ENV.PRESENTER_SCENARIO || 'rainy-saturday';
|
||||
const PRESENTER_AT = __ENV.PRESENTER_AT || '2m30s';
|
||||
const PRESENTER_TIMEOUT = __ENV.PRESENTER_TIMEOUT || '10s';
|
||||
|
||||
const connectionsOpened = new Counter('sse_connections_opened');
|
||||
const connectionsClosed = new Counter('sse_connections_closed');
|
||||
@@ -40,12 +53,18 @@ const eventsReceived = new Counter('sse_events_received');
|
||||
const eventLatency = new Trend('sse_event_latency', true);
|
||||
const scenarioLatency = new Trend('sse_scenario_latency', true);
|
||||
const pingLatency = new Trend('sse_ping_latency', true);
|
||||
const presenterSuccess = new Rate('presenter_post_success');
|
||||
const presenterLatency = new Trend('presenter_post_duration', true);
|
||||
|
||||
const thresholds = {
|
||||
sse_open_success: ['rate>=0.99'],
|
||||
sse_event_latency: ['p(95)<2000'],
|
||||
sse_scenario_latency: ['p(95)<2000'],
|
||||
presenter_post_success: ['rate==1'],
|
||||
sse_connection_errors: ['count==0'],
|
||||
};
|
||||
thresholds[`sse_connections_opened`] = [`count>=${VUS}`];
|
||||
thresholds[`sse_connections_closed`] = [`count>=${VUS}`];
|
||||
|
||||
export const options = {
|
||||
scenarios: {
|
||||
@@ -56,6 +75,15 @@ export const options = {
|
||||
maxDuration: MAX_DURATION,
|
||||
gracefulStop: '30s',
|
||||
},
|
||||
presenter: {
|
||||
executor: 'shared-iterations',
|
||||
vus: 1,
|
||||
iterations: 1,
|
||||
startTime: PRESENTER_AT,
|
||||
maxDuration: '30s',
|
||||
gracefulStop: '5s',
|
||||
exec: 'presenter',
|
||||
},
|
||||
},
|
||||
thresholds,
|
||||
// Every VU owns one long-lived connection; do not pool them.
|
||||
@@ -71,6 +99,39 @@ export function setup() {
|
||||
return { api: API_URL, stream: STREAM_URL, vus: VUS };
|
||||
}
|
||||
|
||||
// One presenter action during the run. This is deliberately separate from the
|
||||
// SSE VUs so the POST does not block a client connection.
|
||||
export function presenter() {
|
||||
if (!CONTROL_USER || !CONTROL_PASS) {
|
||||
throw new Error(
|
||||
'CONTROL_USER and CONTROL_PASS are required for the presenter scenario; source ~/.config/pulsecast/control-auth.env',
|
||||
);
|
||||
}
|
||||
|
||||
const payload = JSON.stringify({
|
||||
scenario: PRESENTER_SCENARIO,
|
||||
weather: { temp_delta_c: 4, precip_mm: 0 },
|
||||
});
|
||||
const basic = encoding.b64encode(`${CONTROL_USER}:${CONTROL_PASS}`);
|
||||
const res = http.post(`${API_URL}/api/scenario`, payload, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Basic ${basic}`,
|
||||
},
|
||||
timeout: PRESENTER_TIMEOUT,
|
||||
tags: { endpoint: 'presenter' },
|
||||
});
|
||||
|
||||
presenterLatency.add(res.timings.duration);
|
||||
const ok = check(res, {
|
||||
'presenter scenario 200': (r) => r.status === 200,
|
||||
});
|
||||
presenterSuccess.add(ok);
|
||||
if (!ok) {
|
||||
console.error(`presenter POST failed: status=${res.status} body=${res.body}`);
|
||||
}
|
||||
}
|
||||
|
||||
export default function () {
|
||||
let opened = false;
|
||||
let closedByUs = false;
|
||||
|
||||
@@ -79,7 +79,7 @@ for name, cm in configmaps.items():
|
||||
required_config = {
|
||||
"pulsecast-shared": {"PUBLIC_WEB_URL", "PUBLIC_API_URL", "API_INTERNAL_URL", "ML_INTERNAL_URL"},
|
||||
"web-config": {"PORT", "NUXT_PUBLIC_API_BASE", "NUXT_API_INTERNAL_BASE"},
|
||||
"api-config": {"PORT", "DATA_DIR", "FASTAPI_URL", "ML_TIMEOUT", "SSE_HEARTBEAT"},
|
||||
"api-config": {"PORT", "FASTAPI_URL", "BOOTSTRAP_REFRESH", "ML_TIMEOUT", "SSE_HEARTBEAT"},
|
||||
"ml-config": {
|
||||
"PORT", "PULSECAST_ROOT", "PULSECAST_ARTIFACTS", "PULSECAST_WEATHER",
|
||||
"PULSECAST_MODELS", "PULSECAST_UPLOADS",
|
||||
@@ -93,6 +93,8 @@ for name, keys in required_config.items():
|
||||
have = set((cm.get("data") or {}).keys())
|
||||
for key in sorted(keys - have):
|
||||
errors.append(f"ConfigMap/{name}: missing key {key}")
|
||||
if "DATA_DIR" in ((configmaps.get("api-config") or {}).get("data") or {}):
|
||||
errors.append("ConfigMap/api-config: DATA_DIR is no longer used by the v1.1 API")
|
||||
|
||||
expected_ports = {"web": 3000, "api": 8080, "ml": 8000, "cloudflared": 2000}
|
||||
expected_uids = {"web": 10001, "api": 65532, "ml": 10001, "cloudflared": 65532}
|
||||
@@ -202,6 +204,11 @@ for name in ("web", "api", "ml", "cloudflared"):
|
||||
if ref not in configmaps:
|
||||
errors.append(f"Deployment/{name}: references missing ConfigMap/{ref}")
|
||||
expected_secret_refs = {
|
||||
"web": {
|
||||
("control-auth", "CONTROL_USER"),
|
||||
("control-auth", "CONTROL_PASS"),
|
||||
("control-auth", "CONTROL_TOKEN"),
|
||||
},
|
||||
"api": {
|
||||
("control-auth", "CONTROL_USER"),
|
||||
("control-auth", "CONTROL_PASS"),
|
||||
|
||||
@@ -216,7 +216,7 @@ for deployment in web api ml cloudflared; do
|
||||
bad "Deployment/$deployment not ready (${ready:-0}/${desired:-?}, $image)"
|
||||
fi
|
||||
if [[ "$deployment" != "cloudflared" && "$image" != *":$TAG" ]]; then
|
||||
warn "Deployment/$deployment image tag is not :$TAG"
|
||||
bad "Deployment/$deployment image is ${image:-<unknown>}, expected :$TAG"
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
Reference in new issue
Block a user