diff --git a/Makefile b/Makefile index c00cd97..c1811c8 100644 --- a/Makefile +++ b/Makefile @@ -16,13 +16,15 @@ WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG) API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG) ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG) -# Service checkouts live beside the main checkout or in per-task worktrees: -# Hackathon_2026/worktrees// +# Service repos are bare Git repos beside this one, each with per-task worktrees: +# Hackathon_2026/web-nuxt.git/feat-web-bootstrap +# Hackathon_2026/api-go.git/feat-api-bootstrap +# Hackathon_2026/ML-py.git/feat-ml-bootstrap # Override any *_DIR on the command line when the layout differs. -SERVICES_ROOT ?= $(if $(wildcard $(abspath $(ROOT)/../../..)/worktrees),$(abspath $(ROOT)/../../..),$(abspath $(ROOT)/..)) -WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/web-nuxt/*) $(wildcard $(SERVICES_ROOT)/web-nuxt)),$(SERVICES_ROOT)/web-nuxt) -API_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/api-go/*) $(wildcard $(SERVICES_ROOT)/api-go)),$(SERVICES_ROOT)/api-go) -ML_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/ML-py/*) $(wildcard $(SERVICES_ROOT)/ML-py)),$(SERVICES_ROOT)/ML-py) +SERVICE_REPOS ?= $(abspath $(ROOT)/../..) +WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/main))) +API_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/main))) +ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main))) WEB_URL ?= https://pulse.noelsabia.dev API_URL ?= https://api.pulse.noelsabia.dev @@ -36,10 +38,13 @@ SSE_HOLD_PINGS ?= 20 SSE_MAX_DURATION ?= 6m K6 ?= K6_BIN ?= $(ROOT)/k6/k6-sse +# xk6-sse v0.2.0 exposes client.on/client.close correctly on k6 v2.0.x but not +# on v2.4.0 (verified 2026-11-14). Pin the known-good k6 version. +K6_BUILD_VERSION ?= v2.0.0 K6_SCRIPT := $(ROOT)/k6/sse.js K6_RESULTS := $(ROOT)/k6/results -.PHONY: help validate validate-kubeconform build build-web build-api build-ml push push-web push-api push-ml set-image deploy deploy-check status logs smoke smoke-local load k6-build secret-commands clean +.PHONY: help validate validate-kubeconform gate1-check build build-web build-api build-ml push push-web push-api push-ml set-image restart deploy deploy-check status logs smoke smoke-local load k6-build secret-commands clean help: ## Show available targets @awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -49,9 +54,9 @@ validate: ## Validate deploy/k8s read-only (schema when possible, never applies) if command -v "$(KUBECONFORM)" >/dev/null 2>&1; then \ echo "== kubeconform (strict, offline schemas)"; \ "$(KUBECONFORM)" -strict -summary -ignore-missing-schemas "$(ROOT)/deploy/k8s"; \ - elif command -v kubectl >/dev/null 2>&1 && kubectl cluster-info >/dev/null 2>&1; then \ + elif command -v kubectl >/dev/null 2>&1 && kubectl --context "$(CONTEXT)" get --raw=/readyz --request-timeout=5s >/dev/null 2>&1; then \ echo "== kubectl client dry-run (read-only)"; \ - kubectl apply --dry-run=client -f "$(ROOT)/deploy/k8s"; \ + $(KUBECTL) apply --dry-run=client -f "$(ROOT)/deploy/k8s"; \ else \ echo "== YAML syntax only: kubeconform is not installed and no cluster is reachable"; \ "$(PYTHON)" -c "import glob, yaml; docs=[d for f in sorted(glob.glob('$(ROOT)/deploy/k8s/*.yaml')) for d in yaml.safe_load_all(open(f)) if d]; assert all(isinstance(d, dict) and d.get('apiVersion') and d.get('kind') for d in docs), 'manifest missing apiVersion/kind'; print(f'{len(docs)} manifests parsed')"; \ @@ -62,6 +67,10 @@ validate-kubeconform: ## Validate schemas with kubeconform (must be installed) @command -v "$(KUBECONFORM)" >/dev/null 2>&1 || { echo "kubeconform not found; go install github.com/yannh/kubeconform/cmd/kubeconform@latest"; exit 1; } "$(KUBECONFORM)" -strict -summary -ignore-missing-schemas "$(ROOT)/deploy/k8s" +gate1-check: ## Read-only Gate 1 readiness: permissions, guardrails, dry-run, Secrets (never mutates) + @NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" REGISTRY="$(REGISTRY)" OWNER="$(OWNER)" TAG="$(TAG)" \ + "$(ROOT)/scripts/gate1-check.sh" + build: build-web build-api build-ml ## Build all three images locally build-web: ## Build the web image @@ -97,12 +106,22 @@ set-image: ## Update live Deployments to TAG (guarded; human only) $(KUBECTL) set image deployment/ml ml="$(ML_IMAGE)" $(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s -deploy-check: ## Verify required Secrets exist (read-only) +restart: ## Force a re-pull/restart of web, api, ml after re-pushing the same tag (guarded) + @test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; } + $(KUBECTL) rollout restart deployment/web deployment/api deployment/ml + $(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s + +deploy-check: ## Verify required Secrets and keys exist (read-only) @for s in gitea-regcred cloudflared control-auth; do \ $(KUBECTL) get secret "$$s" >/dev/null 2>&1 || { \ echo "Missing Secret $$s in $(NAMESPACE); see docs/deploy.md"; exit 1; }; \ done - @echo "required Secrets present" + @for ref in cloudflared:TUNNEL_TOKEN control-auth:CONTROL_USER control-auth:CONTROL_PASS control-auth:CONTROL_TOKEN; do \ + secret="$${ref%%:*}"; key="$${ref#*:}"; \ + value="$$($(KUBECTL) get secret "$$secret" -o jsonpath="{.data.$$key}" 2>/dev/null || true)"; \ + test -n "$$value" || { echo "Secret $$secret is missing key $$key; see docs/deploy.md"; exit 1; }; \ + done + @echo "required Secrets and keys present" deploy: ## Apply manifests (guarded; human only) @test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; } @@ -154,8 +173,11 @@ load: ## Run the 120-client / 5-minute SSE load test k6-build: ## Build a k6 binary with the SSE extension (k6/k6-sse) @command -v go >/dev/null 2>&1 || { echo "Go is required to build xk6"; exit 1; } @cd "$(ROOT)/k6" && GOBIN="$(ROOT)/k6" go install go.k6.io/xk6/cmd/xk6@latest - @cd "$(ROOT)/k6" && "$(ROOT)/k6/xk6" build --with github.com/phymbert/xk6-sse@latest --output "$(ROOT)/k6/k6-sse" - @echo "Built $(ROOT)/k6/k6-sse" + @cd "$(ROOT)/k6" && "$(ROOT)/k6/xk6" build \ + --k6-version "$(K6_BUILD_VERSION)" \ + --with github.com/phymbert/xk6-sse@latest \ + --output "$(ROOT)/k6/k6-sse" + @echo "Built $(ROOT)/k6/k6-sse (k6 $(K6_BUILD_VERSION) + xk6-sse)" secret-commands: ## Print the exact Secret creation commands (creates nothing) @echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic cloudflared \\" @@ -163,7 +185,8 @@ secret-commands: ## Print the exact Secret creation commands (creates nothing) @echo @echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic control-auth \\" @echo " --from-literal=CONTROL_USER='' \\" - @echo " --from-literal=CONTROL_PASS=''" + @echo " --from-literal=CONTROL_PASS='' \\" + @echo " --from-literal=CONTROL_TOKEN=''" @echo @echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret docker-registry gitea-regcred \\" @echo " --docker-server=$(REGISTRY) \\" diff --git a/README.md b/README.md index d56f90c..ace108a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ commands in `docs/deploy.md`. deploy/k8s/ web, api, ml + cloudflared Deployments and Services, ConfigMaps deploy/examples/ example Secret shapes (placeholders only, never applied) scripts/smoke.sh public + in-cluster smoke checks +scripts/gate1-check.sh read-only permissions/manifest/secret readiness check k6/sse.js 120 concurrent SSE clients for 5 minutes docs/deploy.md Gate 1 runbook and exact human checklist docs/load-test.md load-test procedure and how to record results @@ -21,16 +22,34 @@ Makefile validate, build, push, deploy, smoke, load helpers ```bash export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml # chmod 600 +make gate1-check # read-only: permissions, guardrails, dry-run, Secrets make validate # client-side dry run only, never applies +make build # all three images +make push # after docker login to Gitea make deploy CONFIRM=yes # human only; requires the three Secrets first make smoke # public HTTPS + API + ML health checks -make load # k6 SSE load test (needs the k6 SSE extension) +make k6-build # once: pinned k6 + SSE extension +make load # 120 SSE clients for 5 minutes ``` +`make gate1-check` exits 0 when ready, 1 on a hard permission/manifest failure, +and 2 when only human prerequisites (Secrets, Deployments) are pending. It +never mutates the cluster. DNS and public HTTPS are checked separately in the +runbook and by `make smoke`. + Read `docs/deploy.md` before the first deploy. It contains the Gitea token, `docker login`, hello-world push, `gitea-regcred`, Cloudflare Tunnel, public hostname, and verification steps. +## Gate 1 status + +- Manifests validate read-only; `make gate1-check` is green except for pending + human prerequisites. +- All three images build and pass local manifest-equivalent runtime checks + (read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker. +- Still human-only: Gitea read-only token + hello image pull, `gitea-regcred`, + Cloudflare Tunnel + DNS hostnames, and the three Secrets. + ## Git workflow `main` is integration. New work happens on a `feat/` branch in its own @@ -46,3 +65,8 @@ committed directly to `main`. - No secrets, tokens, kubeconfig, or raw CSV in git. - The human applies cluster mutations. `make deploy` refuses to run without `CONFIRM=yes`. +- Every pod sets an explicit numeric `runAsUser`/`runAsGroup`: web/ml + `10001:10001`, api/cloudflared `65532:65532`. SKE rejects a non-numeric image + user under `runAsNonRoot`. +- The tunnel uses HTTP/2 (`--protocol http2`) because TCP/443 to Cloudflare is + verified and UDP/7844 is not. diff --git a/deploy/examples/secrets.example.yaml b/deploy/examples/secrets.example.yaml index e7d0a0f..b7f7c3a 100644 --- a/deploy/examples/secrets.example.yaml +++ b/deploy/examples/secrets.example.yaml @@ -7,7 +7,8 @@ # # kubectl -n fch26-onemanarmy create secret generic control-auth \ # --from-literal=CONTROL_USER='' \ -# --from-literal=CONTROL_PASS='' +# --from-literal=CONTROL_PASS='' \ +# --from-literal=CONTROL_TOKEN='' # # kubectl -n fch26-onemanarmy create secret docker-registry gitea-regcred \ # --docker-server=git.noelsabia.dev \ @@ -31,6 +32,7 @@ type: Opaque stringData: CONTROL_USER: "REPLACE_ME" CONTROL_PASS: "REPLACE_ME" + CONTROL_TOKEN: "REPLACE_ME" --- apiVersion: v1 kind: Secret diff --git a/deploy/k8s/00-config.yaml b/deploy/k8s/00-config.yaml index 7b65398..010ed96 100644 --- a/deploy/k8s/00-config.yaml +++ b/deploy/k8s/00-config.yaml @@ -27,6 +27,7 @@ data: HOST: "0.0.0.0" PORT: "3000" NUXT_TELEMETRY_DISABLED: "1" + TMPDIR: "/tmp" NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev" NUXT_API_INTERNAL_BASE: "http://api:8080" NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream" @@ -45,7 +46,6 @@ data: FASTAPI_URL: "http://ml:8000" ML_TIMEOUT: "1.5s" SSE_HEARTBEAT: "15s" - CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000" --- apiVersion: v1 kind: ConfigMap @@ -56,8 +56,15 @@ metadata: app.kubernetes.io/name: ml app.kubernetes.io/part-of: pulsecast data: + # The image CMD binds 0.0.0.0:8000; PORT documents the contract. PORT: "8000" - DATA_DIR: "/app/data" - MODEL_PATH: "/app/model/model.joblib" + # These are the names pulsecast.config actually reads. The model and JSON + # artifacts are baked into the image; only models/uploads are writable. + PULSECAST_ROOT: "/app" + PULSECAST_ARTIFACTS: "/app/artifacts" + PULSECAST_WEATHER: "/app/artifacts" + PULSECAST_MODELS: "/tmp/pulsecast/models" + PULSECAST_UPLOADS: "/tmp/pulsecast/uploads" + TMPDIR: "/tmp" PYTHONDONTWRITEBYTECODE: "1" PYTHONUNBUFFERED: "1" diff --git a/deploy/k8s/10-ml.yaml b/deploy/k8s/10-ml.yaml index 39ccaeb..8e4d898 100644 --- a/deploy/k8s/10-ml.yaml +++ b/deploy/k8s/10-ml.yaml @@ -28,10 +28,16 @@ spec: securityContext: seccompProfile: type: RuntimeDefault + # Numeric UID is required: a non-numeric image user breaks runAsNonRoot + # (verified on SKE with curlimages/curl, 2026-11-14). + runAsUser: 10001 + runAsGroup: 10001 containers: - name: ml image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1 - imagePullPolicy: IfNotPresent + # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. + # Switch to an immutable tag + IfNotPresent at deploy freeze. + imagePullPolicy: Always ports: - name: http containerPort: 8000 @@ -46,6 +52,12 @@ spec: value: /tmp - name: MPLCONFIGDIR value: /tmp + # Required by FastAPI /train; returns 503 when it is unset. + - name: CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: control-auth + key: CONTROL_TOKEN resources: requests: cpu: 200m diff --git a/deploy/k8s/20-api.yaml b/deploy/k8s/20-api.yaml index aeb4db9..2cf3cf8 100644 --- a/deploy/k8s/20-api.yaml +++ b/deploy/k8s/20-api.yaml @@ -29,10 +29,16 @@ spec: securityContext: seccompProfile: type: RuntimeDefault + # distroless static:nonroot is UID/GID 65532. Keep it numeric so + # runAsNonRoot never depends on the image's user name. + runAsUser: 65532 + runAsGroup: 65532 containers: - name: api image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1 - imagePullPolicy: IfNotPresent + # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. + # Switch to an immutable tag + IfNotPresent at deploy freeze. + imagePullPolicy: Always ports: - name: http containerPort: 8080 @@ -56,6 +62,12 @@ spec: secretKeyRef: name: control-auth key: CONTROL_PASS + # Forwarded as a Bearer token to FastAPI's internal /train routes. + - name: CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: control-auth + key: CONTROL_TOKEN resources: requests: cpu: 100m diff --git a/deploy/k8s/30-web.yaml b/deploy/k8s/30-web.yaml index 41a7557..4871fbe 100644 --- a/deploy/k8s/30-web.yaml +++ b/deploy/k8s/30-web.yaml @@ -31,11 +31,17 @@ spec: securityContext: seccompProfile: type: RuntimeDefault + # Numeric UID: the web image must run as (or at least be readable by) + # 10001:10001. Do not rely on a named image user. + runAsUser: 10001 + runAsGroup: 10001 containers: - name: web # Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`. image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1 - imagePullPolicy: IfNotPresent + # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. + # Switch to an immutable tag + IfNotPresent at deploy freeze. + imagePullPolicy: Always ports: - name: http containerPort: 3000 diff --git a/deploy/k8s/40-cloudflared.yaml b/deploy/k8s/40-cloudflared.yaml index 1672a48..0885257 100644 --- a/deploy/k8s/40-cloudflared.yaml +++ b/deploy/k8s/40-cloudflared.yaml @@ -31,14 +31,22 @@ spec: securityContext: seccompProfile: type: RuntimeDefault + # cloudflared's image user is 65532:65532; set it numerically. + runAsUser: 65532 + runAsGroup: 65532 containers: - name: cloudflared # Public Docker Hub image; no Gitea pull secret needed for this pod. - image: cloudflare/cloudflared:latest + # Pinned rather than :latest so a surprise upstream change cannot break Gate 1. + image: cloudflare/cloudflared:2026.11.0 imagePullPolicy: IfNotPresent args: - tunnel - --no-autoupdate + # SKE smoke confirmed TCP/443 to Cloudflare but did not test + # UDP/7844 (QUIC). HTTP/2 avoids depending on untested UDP. + - --protocol + - http2 - --metrics - 0.0.0.0:2000 - run diff --git a/docs/deploy.md b/docs/deploy.md index 69cc6fa..1cc362c 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -2,150 +2,458 @@ Target: three services (`web`, `api`, `ml`) plus one `cloudflared` Deployment in namespace `fch26-onemanarmy`, reachable over public HTTPS, with Gitea image -pulls working. +pulls proven before the real images are deployed. **The human runs every cluster-mutating command in this document.** The infra agent only writes and validates manifests. No Ingress, cert-manager, external-dns, Jobs, CronJobs, CRDs, Roles, RoleBindings, NetworkPolicies, or other cluster-scoped resources are used. +## What Gate 1 means + +Gate 1 passes only when **all** of these are true: + +1. The cluster pulls an image from Gitea in `fch26-onemanarmy` using + `gitea-regcred` (proven with a hello-world before the real images). +2. A Cloudflare Tunnel serves a public HTTPS URL from this namespace. +3. `web`, `api`, `ml`, and `cloudflared` are Ready. +4. `make smoke SMOKE_SSE=1` passes and a phone on mobile data can open + `https://pulse.noelsabia.dev`. +5. `https://api.pulse.noelsabia.dev/api/healthz` returns HTTP 200. + +Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest +dry-run, or a container that only runs on your laptop. + +## Current external status (2026-11-14 03:50 UTC smoke) + +Findings from `CLUSTER-SMOKE.md`; re-check before trusting this snapshot: + +- **Verified:** cluster access, scheduling, Service DNS/HTTP, probes, Secrets, + non-root + read-only-root + `emptyDir`, and egress to Cloudflare, + Open-Meteo, and `git.noelsabia.dev`. +- **No public DNS yet:** `pulse.noelsabia.dev` and + `api.pulse.noelsabia.dev` have no records. That is expected until the + Cloudflare Tunnel public hostnames are created. +- **No tunnel yet:** Cloudflare Tunnel has not been created. +- **Gitea pull auth is untested:** the network path works, but no + `gitea-regcred` image pull has succeeded because the read-only token does + not exist yet. The hello pod in Step 2 is the first proof. +- **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the + `cloudflared` manifest uses `--protocol http2` instead of relying on + unverified UDP. +- **Numeric UIDs are required:** a non-numeric image user breaks + `runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in + `deploy/k8s/` sets an explicit numeric `runAsUser`/`runAsGroup`. +- **Repo status (2026-11-14 ~05:10 UTC):** all three images now build and pass + local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp` + tmpfs). There is no remaining Dockerfile blocker; the human Gitea/Tunnel + steps are what remain. + +## What is possible now vs what is blocked + +| Step | Who | Status | +|---|---|---| +| Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) | +| Gitea push login + hello-world push | human | **ready now**; registry auth still untested | +| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **ready now** (needs token) | +| Cloudflare Tunnel + public hostnames/DNS | human | **blocked on human** — not created; DNS absent | +| `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally | +| `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally | +| `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally | +| Full deploy + smoke + Gate 1 sign-off | human | **ready once the Gitea/Tunnel steps are done** | + +`make build` and `make push` cover all three services; no Dockerfile is +missing now. Do not deploy placeholder images under the real service names. + +## Before you start + +Run every block in this document in **one shell**, from this worktree. The +commands are bash/zsh compatible. + +```bash +cd /home/noel/Documents/Hackathon_2026/infra-k8.git/feat-infra-bootstrap + +export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarmy-sa-kubeconfig.yaml" +export NS="fch26-onemanarmy" +export CTX="fch26-onemanarmy-sa-context" +export REGISTRY="git.noelsabia.dev" +export OWNER="noelsabia" +export TAG="gate1" + +chmod 600 "$KUBECONFIG" +``` + +The kubeconfig is **not in this repo**. If it moved, change the path; never +commit it or paste its contents anywhere. + +Check the local tools: + +```bash +for tool in kubectl docker curl jq make; do + command -v "$tool" >/dev/null || echo "MISSING: $tool" +done +``` + +Expected: no `MISSING` lines. `kubeconform`, `k6`, and `yamllint` are optional. + +Check the context and the namespace permissions (read-only): + +```bash +kubectl config current-context +kubectl --context "$CTX" -n "$NS" auth can-i create deployments +kubectl --context "$CTX" -n "$NS" auth can-i create services +kubectl --context "$CTX" -n "$NS" auth can-i create ingresses +kubectl --context "$CTX" -n "$NS" auth can-i create jobs +kubectl --context "$CTX" -n "$NS" get deployments +``` + +Expected: + +```text +fch26-onemanarmy-sa-context +yes +yes +no +no +No resources found in fch26-onemanarmy namespace. +``` + +If `deployments` or `services` is not `yes`, or `ingresses`/`jobs` is not +`no`, stop and report. Do not work around permissions with cluster-scoped +resources. + ## Gate 1 checklist Run top to bottom. Stop at the first failure and use [Troubleshooting](#troubleshooting). -- [ ] 1. Kubeconfig exported, `600`, correct context, namespace reachable. -- [ ] 2. Gitea **read-only package token** created and `docker login` succeeds. -- [ ] 3. Hello-world image pushed and pulled by the cluster with `gitea-regcred`. -- [ ] 4. Secrets created: `gitea-regcred`, `cloudflared`, `control-auth`. -- [ ] 5. Real `web`/`api`/`ml` images built, tagged `gate1`, pushed to Gitea. -- [ ] 6. `make validate` passes locally. -- [ ] 7. Cloudflare Tunnel exists; both public hostnames configured. -- [ ] 8. `make deploy CONFIRM=yes` completes and all four Deployments are ready. -- [ ] 9. `make smoke SMOKE_SSE=1` passes. -- [ ] 10. `https://pulse.noelsabia.dev` opens on a phone over mobile data. -- [ ] 11. `https://api.pulse.noelsabia.dev/api/healthz` returns 200. -- [ ] 12. Record the deployed image tags and the time in the status note. +- [ ] 0. `make gate1-check` shows 0 hard failures. +- [ ] 1. Gitea push login succeeds (`Login Succeeded`). +- [ ] 2. Read-only package token created; hello image pushed to Gitea. +- [ ] 3. `gitea-regcred` created; hello pod is `1/1 Running` from Gitea + (the Secret existing is not proof). +- [ ] 4. Cloudflare Tunnel created; both public hostnames configured and DNS + records resolve. +- [ ] 5. `cloudflared` Secret created; logs show `Registered tunnel connection`. +- [ ] 6. Public hello route returns HTTP 200 (proves tunnel + DNS + TLS). +- [ ] 7. `control-auth` Secret created. +- [ ] 8. All three images built, tagged `gate1`, and pushed. +- [ ] 9. `make validate` and `make gate1-check` pass. +- [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready. +- [ ] 11. `make smoke SMOKE_SSE=1` passes. +- [ ] 12. Phone over mobile data opens `https://pulse.noelsabia.dev`; + `https://api.pulse.noelsabia.dev/api/healthz` returns 200. +- [ ] 13. Image tags/digests and the time are recorded. -## 0. Prerequisites - -The kubeconfig is **not in this repo**. It lives outside git: +## Step 0 — read-only readiness and dry-run ```bash -export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarmy-sa-kubeconfig.yaml" -export NS=fch26-onemanarmy -export CTX=fch26-onemanarmy-sa-context -export REGISTRY=git.noelsabia.dev -export OWNER=noelsabia -export TAG=gate1 - -chmod 600 "$KUBECONFIG" +make validate +make gate1-check ``` -Verify read-only first: +Expected now (before any human work): + +- `make validate` prints `== kubectl client dry-run (read-only)` and 11 + `created (dry run)` lines. +- `make gate1-check` reports **0 hard failures**, the expected permission + matrix, and lists missing Secrets/Deployments as `[pending]`. +- Exit code 2 from `make gate1-check` is correct at this stage: it means only + human prerequisites remain, not that the manifests are broken. + +Exit codes: `0` ready, `1` hard failure (permissions or manifests), `2` only +pending human prerequisites. Use `make gate1-check ALLOW_PENDING=1` when you +want exit 0 while prerequisites are still being prepared. + +Neither target mutates the cluster. `make validate` uses +`kubectl apply --dry-run=client`; `make gate1-check` only reads +configuration/RBAC/secrets and runs the same dry-run. + +## Step 1 — Gitea credentials + +You need **two different credentials**. Do not reuse the read-only token for +pushing. + +1. **Push credential:** your normal Gitea account password, or a token with + `write:package`. Use a token if the account has 2FA. +2. **Cluster pull credential:** a separate token with **only** + `read:package`. This becomes `gitea-regcred`; it never needs write access. + +Create the read-only token in Gitea: **Settings → Applications → Generate New +Token** (or the package-token screen). Name it `pulsecast-k8s-pull`; scope +`read:package`. Copy it once. + +Log in for pushing: ```bash -kubectl config current-context # fch26-onemanarmy-sa-context -kubectl -n "$NS" auth can-i create deployments # yes -kubectl -n "$NS" auth can-i create services # yes -kubectl -n "$NS" auth can-i create ingresses # no -kubectl -n "$NS" auth can-i create jobs # no -kubectl -n "$NS" get pods,svc,secrets +read -rsp 'Gitea push password or write:package token: ' GITEA_PUSH_SECRET; echo +printf '%s' "$GITEA_PUSH_SECRET" | docker login "$REGISTRY" -u "$OWNER" --password-stdin ``` -If any of the first three answers differ, stop and report; do not work around -permissions with cluster-scoped resources. +Expected: -Local tools: `git`, `make`, `bash`, `curl`, `docker`, `kubectl`. Optional: -`kubeconform` (schema validation), `jq`, `k6` built with the SSE extension. +```text +Login Succeeded +``` -## 1. Gitea read-only package token - -In Gitea: **Settings → Applications → Generate New Token** (or the package -token screen). Scope: read-only package access. Do not reuse the admin token. +Store the read-only token in a shell variable for the Secret step: ```bash -read -rsp 'Gitea package token: ' GITEA_TOKEN; echo -printf '%s' "$GITEA_TOKEN" | docker login "$REGISTRY" -u "$OWNER" --password-stdin +read -rsp 'Gitea read-only package token: ' GITEA_RO_TOKEN; echo ``` -A successful login prints `Login Succeeded`. Never put the token in a file, -a command that lands in shell history, or this repo. +Expected: no output (the value is not echoed). -## 2. Hello-world image pull (prove the registry path) +If login fails with `401 Unauthorized`, the password/token or username is +wrong, or 2FA requires a token. Do not put either credential in a file, a +command that lands in shell history, or this repo. + +## Step 2 — hello-world image push and pull + +This proves the registry path before spending time on application images. ```bash docker pull nginxinc/nginx-unprivileged:alpine docker tag nginxinc/nginx-unprivileged:alpine "$REGISTRY/$OWNER/hello:gate1" docker push "$REGISTRY/$OWNER/hello:gate1" - -kubectl -n "$NS" create secret docker-registry gitea-regcred \ - --docker-server="$REGISTRY" \ - --docker-username="$OWNER" \ - --docker-password="$GITEA_TOKEN" - -kubectl -n "$NS" create deployment hello \ - --image="$REGISTRY/$OWNER/hello:gate1" -kubectl -n "$NS" patch deployment hello --type=merge -p \ - '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"gitea-regcred"}]}}}}' -kubectl -n "$NS" rollout status deployment/hello --timeout=180s -kubectl -n "$NS" get pods -l app=hello ``` -Pass: the pod is `Running` and `kubectl describe pod` shows the image pulled. -Clean up before the real deploy: +Expected: the push ends with a digest line, for example: + +```text +gate1: digest: sha256:... size: ... +``` + +If you see `denied: requested access to the resource is denied`, the push +credential lacks `write:package` or the owner/package path is wrong. Fix that +before continuing. + +Create the pull secret with the **read-only** token: ```bash -kubectl -n "$NS" delete deployment hello +kubectl --context "$CTX" -n "$NS" create secret docker-registry gitea-regcred \ + --docker-server="$REGISTRY" \ + --docker-username="$OWNER" \ + --docker-password="$GITEA_RO_TOKEN" ``` -If this fails, fix the token/secret before building the application images. +Expected: `secret/gitea-regcred created`. If it already exists, delete and +recreate it after reading the troubleshooting note: +`kubectl --context "$CTX" -n "$NS" delete secret gitea-regcred`. -## 3. Create the remaining Secrets +Deploy the hello image and make the cluster pull it: -Generate values locally; never commit them. +```bash +kubectl --context "$CTX" -n "$NS" create deployment hello \ + --image="$REGISTRY/$OWNER/hello:gate1" + +kubectl --context "$CTX" -n "$NS" patch deployment hello --type=merge -p \ + '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"gitea-regcred"}]}}}}' + +kubectl --context "$CTX" -n "$NS" rollout status deployment/hello --timeout=180s +kubectl --context "$CTX" -n "$NS" get pods -l app=hello -o wide +``` + +Expected: + +```text +deployment "hello" successfully rolled out +NAME READY STATUS RESTARTS AGE +hello-xxxxxxxxxx-xxxxx 1/1 Running 0 20s +``` + +**This is the first real pass/fail.** A `gitea-regcred` Secret existing is not +proof: the Gate 1 Gitea item is complete only when this hello pod is `1/1 +Running` after pulling from Gitea. If the pod is not `Running`, stop. +`ImagePullBackOff` means the token, secret namespace, image tag, or registry +DNS is wrong. Diagnose with: + +```bash +kubectl --context "$CTX" -n "$NS" describe pod -l app=hello +``` + +Keep the hello Deployment until Step 6 proves the public route. Create a +Service for it so Cloudflare has something to reach: + +```bash +kubectl --context "$CTX" -n "$NS" expose deployment hello \ + --port=8080 --target-port=8080 +``` + +Expected: `service/hello exposed`. + +## Step 3 — Cloudflare Tunnel + +As of the 03:50 UTC smoke, neither public hostname has a DNS record. That is +expected before the tunnel route exists; do not add manual A/AAAA records. + +```bash +getent hosts pulse.noelsabia.dev || echo 'pulse: no DNS record yet (expected)' +getent hosts api.pulse.noelsabia.dev || echo 'api: no DNS record yet (expected)' +``` + +1. Open Cloudflare **Zero Trust → Networks → Tunnels → Create a tunnel**. +2. Connector: **Cloudflared**. Name: `pulsecast-gate1`. +3. Save and copy the tunnel token. It is shown once. +4. Open the tunnel's **Public Hostnames** tab and add: + + | Public hostname | Service | + |---|---| + | `pulse.noelsabia.dev` | `http://hello:8080` (temporary, Gate 1-A) | + | `api.pulse.noelsabia.dev` | `http://api:8080` (works after Step 10) | + + For the final Gate 1-B route, change `pulse.noelsabia.dev` to + `http://web:3000` after the web image is deployed. If you already have the + web image, you may set `web:3000` now and skip the hello route; then the + public hello check in Step 6 is not available, but the tunnel proof still + happens when the real web pod is Ready. + +5. The hostnames must be in the Cloudflare zone for `noelsabia.dev`. + Cloudflare creates the DNS records for a tunnel hostname; do **not** add + manual A/AAAA records. If the dashboard refuses to save a hostname because + a conflicting A/AAAA/CNAME already exists for that exact name, delete the + conflicting record first. +6. Do not add an Ingress. The tunnel is the only public path. + +After saving both hostnames, DNS should resolve to Cloudflare: + +```bash +getent hosts pulse.noelsabia.dev +getent hosts api.pulse.noelsabia.dev +``` + +Expected: one or more Cloudflare IPv4/IPv6 addresses. If there is still no +record, the public hostname was not saved, it is in the wrong zone, or a +conflicting record exists. Fix that in the dashboard before testing HTTPS; +`curl` cannot succeed without DNS. + +Create the tunnel Secret (the token is not echoed): + +```bash +read -rsp 'Cloudflare tunnel token: ' TUNNEL_TOKEN; echo +kubectl --context "$CTX" -n "$NS" create secret generic cloudflared \ + --from-literal=TUNNEL_TOKEN="$TUNNEL_TOKEN" +``` + +Expected: `secret/cloudflared created`. + +Deploy only the tunnel first. This does not need the application images: + +```bash +kubectl --context "$CTX" -n "$NS" apply -f deploy/k8s/40-cloudflared.yaml +kubectl --context "$CTX" -n "$NS" rollout status deployment/cloudflared --timeout=180s +kubectl --context "$CTX" -n "$NS" logs deployment/cloudflared --tail=100 \ + | grep 'Registered tunnel connection' || echo 'NO TUNNEL CONNECTION' +``` + +Expected: + +```text +deployment.apps/cloudflared created +deployment "cloudflared" successfully rolled out +... Registered tunnel connection ... +``` + +(Usually four `Registered tunnel connection` lines, one per edge.) + +The manifest deliberately uses `--protocol http2`: TCP/443 to Cloudflare is +verified, UDP/7844 (QUIC) is not. Do not switch to QUIC during Gate 1. If you +do test QUIC later, revert to HTTP/2 on any connection error. + +If `cloudflared` is in `CrashLoopBackOff`, the token is invalid/expired or the +tunnel was deleted: + +```bash +kubectl --context "$CTX" -n "$NS" logs deployment/cloudflared --previous +kubectl --context "$CTX" -n "$NS" describe pod -l app.kubernetes.io/name=cloudflared +``` + +## Step 4 — control-auth Secret + +The API reads `CONTROL_USER` and `CONTROL_PASS` for browser control auth. +Both API and ML also need the same `CONTROL_TOKEN`: the API forwards it as a +Bearer token to FastAPI's internal `/train` routes. Generate both locally; +never commit them. ```bash read -rsp 'Control password: ' CONTROL_PASS; echo -kubectl -n "$NS" create secret generic control-auth \ +CONTROL_TOKEN="$(python3 -c 'import secrets; print(secrets.token_hex(24))')" +kubectl --context "$CTX" -n "$NS" create secret generic control-auth \ --from-literal=CONTROL_USER=control \ - --from-literal=CONTROL_PASS="$CONTROL_PASS" - -# Token from Cloudflare Zero Trust → Networks → Tunnels → your tunnel. -read -rsp 'Cloudflare tunnel token: ' TUNNEL_TOKEN; echo -kubectl -n "$NS" create secret generic cloudflared \ - --from-literal=TUNNEL_TOKEN="$TUNNEL_TOKEN" - -kubectl -n "$NS" get secrets + --from-literal=CONTROL_PASS="$CONTROL_PASS" \ + --from-literal=CONTROL_TOKEN="$CONTROL_TOKEN" ``` -`make secret-commands` prints the same commands with placeholders. +Expected: `secret/control-auth created`. -## 4. Build and push the three images +The token is shared by API and ML, not the browser password. Without it +FastAPI `/train` returns 503 and the API's `/api/train` proxy cannot work. -The manifests expect: +`make secret-commands` prints all three Secret commands with placeholders; it +creates nothing. -| Service | Image | Port | Health paths | -|---|---|---|---| -| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `/` | -| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `/api/healthz`, `/api/readyz` | -| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `/healthz`, `/readyz` | +## Step 5 — build and push the three images (Phase B) + +Check whether the Dockerfiles exist: ```bash -make build # builds all three from the service worktrees -make push # docker push to Gitea +test -f ../../web-nuxt.git/feat-web-bootstrap/Dockerfile && echo web-ok || echo web-blocked +test -f ../../api-go.git/feat-api-bootstrap/Dockerfile && echo api-ok || echo api-blocked +test -f ../../ML-py.git/feat-ml-bootstrap/Dockerfile && echo ml-ok || echo ml-blocked ``` -`make build` auto-detects the service worktrees under -`/home/noel/Documents/Hackathon_2026/worktrees/`. Override when needed: +Expected **now** (all three Dockerfiles landed on 2026-11-14): + +```text +web-ok +api-ok +ml-ok +``` + +Build all three: + +```bash +make build +``` + +All three images were verified locally with the manifest's runtime +constraints: + +- API: distroless `static:nonroot`, `--user 65532:65532`, read-only root, + `/tmp` tmpfs; `/api/healthz` and `/api/readyz` returned 200. +- ML: image `USER app` is non-numeric, but the manifest overrides it with + `runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/healthz` and + `/readyz` returned 200 and `/meta` loaded the baked model. +- Web: image `USER node` is non-numeric, but the manifest overrides it with + `runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/` returned + HTTP 200 and the runtime `NUXT_PUBLIC_API_BASE` appeared in the HTML. +- No image was pushed or deployed by the infra agent. + +Push: + +```bash +make push +``` + +Expected: each `docker push` ends with a digest. If authentication expired, +repeat the push login from Step 1. + +The Makefile auto-detects the service worktrees: + +```text +../../web-nuxt.git/feat-web-bootstrap +../../api-go.git/feat-api-bootstrap +../../ML-py.git/feat-ml-bootstrap +``` + +Override when the layout differs: ```bash make build WEB_DIR=/path/to/web-nuxt API_DIR=/path/to/api-go ML_DIR=/path/to/ML-py ``` -If `OWNER` or `TAG` differs from the manifest defaults, either update -`deploy/k8s/*.yaml` before `make deploy`, or deploy first and then run the -guarded image update: +If `OWNER` or `TAG` differs from the manifests, update `deploy/k8s/*.yaml` +before `make deploy`, or deploy first and then run the guarded image update: ```bash make set-image CONFIRM=yes OWNER= TAG= @@ -155,171 +463,281 @@ make set-image CONFIRM=yes OWNER= TAG= Each service image must: -- listen on `0.0.0.0` at the port above; -- run as a non-root user (numeric UID, no shell required); +- listen on `0.0.0.0` at the port below; +- run as a **non-root numeric UID** (no named user, no shell required); - write only to `/tmp` (the pod root filesystem is read-only); - start with the environment in `deploy/k8s/00-config.yaml`; - be self-contained: no runtime download of data, model, fonts, or packages. +| Service | Image | Port | UID:GID | Health paths | +|---|---|---|---|---| +| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` | +| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` | +| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` | + +The manifests set `runAsUser`/`runAsGroup` explicitly, so the image's named +user is irrelevant; the files must be readable by that numeric UID. `api` +uses distroless `static:nonroot` (65532); web and ml use 10001. If a Dockerfile +genuinely needs another UID, change the manifest and this table together, and +keep the value numeric. + `api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`, -`CONTROL_USER`, `CONTROL_PASS`, `SSE_HEARTBEAT`, and `ML_TIMEOUT`. -`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in or -provided at runtime; `ml` must find its model and artifacts under the paths its -Dockerfile defines. The env names in the manifests are the integration point: -change them here first if a service needs a different name. +`CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and +`ML_TIMEOUT`. +`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in +or read it at runtime from the ConfigMap; the browser must use the public API +hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`, +`PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and +`CONTROL_TOKEN`; its image CMD binds `0.0.0.0:8000`. The env names in the +manifests are the integration point: change them here first if a service needs +a different name. The ML image currently uses a named `app` user; the manifest +overrides it with numeric UID 10001, which was verified locally. -## 5. Cloudflare Tunnel +## Step 6 — prove public HTTPS with hello -1. Zero Trust → **Networks → Tunnels → Create a tunnel**. -2. Connector: **Cloudflared**. Name: `pulsecast-gate1`. -3. Copy the tunnel token (it is shown once). Store it in the `cloudflared` - Secret (step 3). -4. Add public hostnames: +Do this **before** deleting hello. It proves tunnel + DNS + TLS without +depending on web/ml. - | Public hostname | Service | - |---|---| - | `pulse.noelsabia.dev` | `http://web:3000` | - | `api.pulse.noelsabia.dev` | `http://api:8080` | +```bash +curl -fsS -o /dev/null -w 'hello %{http_code}\n' https://pulse.noelsabia.dev/ +``` - The hostnames must be in the Cloudflare zone for `noelsabia.dev`. The - service URL uses the Kubernetes Service name and port inside the namespace; - no Ingress is involved. -5. For the first smoke test before the app images exist, temporarily point - `pulse.noelsabia.dev` at `http://hello:8080`, then switch it to - `http://web:3000` after step 7. +Expected: `hello 200`. -DNS records for the public hostnames are created by Cloudflare; do not add -manual A/AAAA records for a tunnel hostname. +If you get `502`, `530`, or `1033`, the edge cannot reach the Service. Check: -## 6. Validate before applying +```bash +kubectl --context "$CTX" -n "$NS" get endpoints hello +kubectl --context "$CTX" -n "$NS" get svc hello +kubectl --context "$CTX" -n "$NS" logs deployment/cloudflared --tail=50 +``` + +The hostname route must be `http://hello:8080` and the hello Service must have +an endpoint. Fix the dashboard route or the Service; do not add an Ingress. + +After the public hello check passes, delete it so it cannot mask a web problem: + +```bash +kubectl --context "$CTX" -n "$NS" delete service hello +kubectl --context "$CTX" -n "$NS" delete deployment hello +``` + +Expected: `service "hello" deleted` and `deployment.apps "hello" deleted`. + +`https://pulse.noelsabia.dev` will now return an edge error until the real web +Deployment is Ready. That is expected; do not "fix" it with an Ingress. + +## Step 7 — validate and check readiness again ```bash make validate +make gate1-check ``` -This is read-only: +Expected once Secrets and Deployments exist: -- `kubeconform -strict` validates the resource schemas if installed; -- otherwise, if a cluster is reachable, `kubectl apply --dry-run=client` - validates against the API server; -- if neither is available it parses the YAML and prints a warning that schema - validation was skipped. +- `make validate`: 11 resources created (dry run); +- `make gate1-check`: 0 hard failures, 0 pending, `RESULT: READY for Gate 1` + (exit 0). -Never run `kubectl apply` / `delete` from an agent session. The commands below -are for the human. +Before the deploy, pending Deployments keep it at exit 2. That is fine. -## 7. Deploy +## Step 8 — deploy ```bash -make deploy-check # read-only: required Secrets exist -make deploy CONFIRM=yes # applies deploy/k8s and waits for rollouts +make deploy-check +make deploy CONFIRM=yes +``` + +Expected: + +```text +required Secrets and keys present +... created/configured ... +deployment "ml" successfully rolled out +deployment "api" successfully rolled out +deployment "web" successfully rolled out +deployment "cloudflared" successfully rolled out ``` `make deploy` fails closed: without `CONFIRM=yes` it changes nothing. It does -not create the namespace or secrets. +not create the namespace or Secrets. It applies all 11 manifests, including +the ConfigMaps. -If the three service images are not available yet, deploy only the tunnel and -use the hello service to prove public HTTPS; return to this step when the real -images exist. Do not deploy a placeholder image under the real service names. +If the real images do not exist, the rollouts fail with `ImagePullBackOff`. +Stop and finish Step 5; do not deploy a placeholder under `web`, `api`, or +`ml`. -Check the result: +Check the state: ```bash -kubectl -n "$NS" get deploy,pods,svc -kubectl -n "$NS" get pods -o wide -kubectl -n "$NS" logs deployment/cloudflared --tail=50 | grep 'Registered tunnel connection' +make status +kubectl --context "$CTX" -n "$NS" get pods -o wide +kubectl --context "$CTX" -n "$NS" logs deployment/cloudflared --tail=50 \ + | grep 'Registered tunnel connection' || echo 'NO TUNNEL CONNECTION' ``` -Expected: `web`, `api`, `ml`, `cloudflared` each `1/1`; Services `web`, `api`, -`ml` are `ClusterIP`; `cloudflared` has no Service (it dials out). +Expected: `web`, `api`, `ml`, and `cloudflared` each `1/1`; Services `web`, +`api`, and `ml` are `ClusterIP`; `cloudflared` has no Service. -## 8. Verify +### Re-pushing the same tag -Public checks (also runnable from a phone on mobile data): +The manifests use `imagePullPolicy: Always` while the tag stays `gate1`. After +rebuilding and pushing the same tag, force a re-pull: ```bash -curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/ -curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz -curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz -curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}' -curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20 +make restart CONFIRM=yes ``` -The SSE output must start with `retry: 3000`, contain a `scenario` event -immediately, and show a `ping` within 20 s. +This restarts `web`, `api`, and `ml` and waits for the rollout. Do not use it +during the demo window unless you are fixing a blocker. -Full smoke test, including the ML service through a temporary -`kubectl port-forward`: +## Step 9 — verify public HTTPS and SSE + +Full smoke, including the ML service through a short `kubectl port-forward`: ```bash make smoke SMOKE_SSE=1 ``` -Local variant once the three dev servers are running: +Expected: each check prints `PASS`, the summary ends with `0 failed`, and TLS +verification passes (`ssl_verify_result=0`). If the ML port-forward cannot +start, set `ML_URL` or `SKIP_ML=1`; do not silently drop the ML check from the +sign-off. + +Manual public checks (also run from a phone on mobile data): ```bash -make smoke-local +curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/ +curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz +curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz +curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}' +curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20 ``` -In-cluster connectivity (optional; creates and removes one short-lived pod): +Expected: + +```text +web 200 +api 200 +ready 200 +{ + "data_version": "...", + "model_version": "..." +} +retry: 3000 + +event: scenario +id: ... +data: {...} + +event: ping +data: {"t":"..."} +``` + +The SSE output must start with `retry: 3000`, contain a `scenario` event +immediately, and show a `ping` within 20 seconds. If events are delayed +through Cloudflare, see the SSE row in Troubleshooting. + +In-cluster connectivity (optional; **creates and removes one short-lived +pod**, so the human runs it): ```bash -kubectl -n "$NS" run gate1-netcheck --rm -it --restart=Never \ - --image=curlimages/curl:latest --command -- sh -c \ +kubectl --context "$CTX" -n "$NS" run gate1-netcheck --rm -i --restart=Never \ + --image=curlimages/curl:latest \ + --overrides='{"spec":{"securityContext":{"runAsNonRoot":true,"runAsUser":1000,"runAsGroup":1000}}}' \ + --command -- sh -c \ 'curl -fsS http://api:8080/api/healthz && \ curl -fsS http://ml:8000/healthz && \ curl -fsS http://web:3000/ >/dev/null && echo connectivity-ok' ``` -TLS is verified by `make smoke` (`ssl_verify_result=0`). Certificates are -issued by Cloudflare at the edge; there is no cert-manager resource. +Expected: `{"ok":true}{"ok":true}connectivity-ok` (order may vary with the +shell output). If the namespace's Pod Security Admission blocks this ad-hoc +pod, skip it and use `kubectl exec deploy/web -- ...` or the smoke test; do not +weaken the application Deployments to make a diagnostic pod run. -## 9. Record the Gate 1 result +Finally, confirm the checker is green end-to-end: -Write down, in the status note or README: +```bash +make gate1-check CHECK_PUBLIC=1 +``` -- deploy time and commit hash; -- image tags actually deployed (`kubectl -n "$NS" get deploy -o jsonpath=...`); -- output of `make smoke SMOKE_SSE=1`; -- public URLs and the device/network used; +Expected: `RESULT: READY for Gate 1` (exit 0). + +## Step 10 — record the Gate 1 result + +Record in the status note, README, or commit message: + +```bash +date -u +'%Y-%m-%dT%H:%M:%SZ' +git rev-parse HEAD +kubectl --context "$CTX" -n "$NS" get deploy -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.template.spec.containers[0].image}{"\n"}{end}' +``` + +Also save: + +- the output of `make smoke SMOKE_SSE=1`; +- the public URLs and the device/network used (phone on mobile data); - any warning that remains. +Image digests are in the `docker push` output; tags alone are not enough if +you need to reproduce the exact deploy. + ## Troubleshooting | Symptom | Check | Fix | |---|---|---| -| `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod `; `kubectl -n "$NS" get secret gitea-regcred` | Token is read-only package scoped; image tag exists; secret is in the same namespace. | -| `CreateContainerConfigError` | `kubectl -n "$NS" describe pod ` | A referenced Secret key is missing (`control-auth` or `cloudflared`). Recreate the Secret. | -| `cloudflared` CrashLoopBackOff | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. | -| Public 502 / 1033 | `kubectl -n "$NS" get endpoints web api`; cloudflared logs | Service has no ready endpoints, wrong port, or hostname route points at the wrong service. | -| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `kubectl -n "$NS" exec deploy/api -- ls /app/data` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. | -| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds 120 s or the model path is wrong. Fix the image/path; startupProbe allows 120 s. | -| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing build-time API base or server error. Rebuild with `NUXT_PUBLIC_API_BASE`. | -| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering; the web client falls back to polling `/api/scenario` every 5 s. Keep the heartbeat at 15 s. | -| `make deploy-check` fails | `kubectl -n "$NS" get secrets` | Create the missing Secret; do not commit it. | +| `docker login` → `401 Unauthorized` | Username and credential; 2FA | Use a token with `write:package` (not the read-only token), or the account password if 2FA is off. | +| `docker push` → `denied: requested access to the resource is denied` | Push credential scope; owner/package path | Use a `write:package` token; make sure `$OWNER` matches the Gitea image owner. | +| `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod -l app=hello` (and the real pod); `kubectl -n "$NS" get secret gitea-regcred` | The token must have `read:package`; the Secret must be in `fch26-onemanarmy`; the tag must exist in Gitea. Do not print the secret data. | +| `CreateContainerConfigError` | `kubectl -n "$NS" describe pod ` | A referenced Secret key is missing (`control-auth`: `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`) or the `cloudflared` Secret lacks `TUNNEL_TOKEN`. Recreate the Secret. | +| `container has runAsNonRoot and image will run as root` or a non-numeric-user error | `kubectl -n "$NS" get pod -o yaml`; inspect `securityContext` | Keep `runAsNonRoot: true` and set a positive numeric `runAsUser`/`runAsGroup` (SKE caveat). The manifests already do; do not "fix" this by disabling `runAsNonRoot`. | +| Public hostname has no DNS record | `getent hosts pulse.noelsabia.dev` / `getent hosts api.pulse.noelsabia.dev` | Save the public hostname in the tunnel dashboard in the `noelsabia.dev` zone; remove a conflicting A/AAAA/CNAME for that exact name. Do not add manual records for a tunnel hostname. | +| `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. | +| `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. | +| Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. | +| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api.pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. | +| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. | +| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. | +| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. | +| `make gate1-check` exits 2 | Output lists `[pending]` Secrets/Deployments | Complete the human steps above. Exit 2 is not a manifest failure. | +| `make deploy` refuses | `CONFIRM` is empty | Re-run `make deploy CONFIRM=yes`; this is the safety guard. | +| `kubectl` → `Forbidden` | `kubectl config current-context` | Use `--context "$CTX"`. Do not create Roles/Ingress/Jobs as a workaround; escalate instead. | +| `make validate` fails | Dry-run output | Fix the manifest; do not apply it to see whether it works. | ## Rollback No cluster-scoped resources are created, so rollback is namespace-local: ```bash -kubectl -n "$NS" rollout undo deployment/web -kubectl -n "$NS" rollout undo deployment/api -kubectl -n "$NS" rollout undo deployment/ml -kubectl -n "$NS" rollout undo deployment/cloudflared -kubectl -n "$NS" rollout status deployment/web --timeout=180s +kubectl --context "$CTX" -n "$NS" rollout undo deployment/web +kubectl --context "$CTX" -n "$NS" rollout undo deployment/api +kubectl --context "$CTX" -n "$NS" rollout undo deployment/ml +kubectl --context "$CTX" -n "$NS" rollout undo deployment/cloudflared +kubectl --context "$CTX" -n "$NS" rollout status deployment/web --timeout=180s ``` If the tunnel token or route is wrong, fix the Secret / Cloudflare dashboard -and restart `cloudflared`; do not create an Ingress as a workaround. +and restart `cloudflared`; do not create an Ingress as a workaround. Delete any +leftover hello resources: + +```bash +kubectl --context "$CTX" -n "$NS" delete service hello --ignore-not-found +kubectl --context "$CTX" -n "$NS" delete deployment hello --ignore-not-found +``` ## Security notes - Never commit the kubeconfig, tunnel token, control password, Gitea token, or any `.env` file. `.gitignore` covers the usual filenames; check `git status` before every commit. +- The kubeconfig must stay at mode `600` and outside the repos. It was shared + in chat: rotate/revoke the service-account token after the event. - Rotate the Gitea package token and the control password after the event. - Rotate the service-account token after the event; it was shared in chat. -- The kubeconfig must stay at mode `600` and outside the repos. -- `make smoke` prints no secrets; do not add `set -x` to the scripts with a - live `KUBECONFIG`. +- Use the read-only package token only for `gitea-regcred`; use a separate + write credential for pushes. +- `make smoke` and `make gate1-check` print no secrets; do not add `set -x` to + scripts while a live `KUBECONFIG` or token is exported. +- Stop the tunnel and rotate the Gitea token if either leaks. Deleting the + tunnel in Cloudflare invalidates its token. diff --git a/docs/load-test.md b/docs/load-test.md index 4133251..b56d9ef 100644 --- a/docs/load-test.md +++ b/docs/load-test.md @@ -7,17 +7,22 @@ and reports connection count plus p95 event latency. Native k6 buffers a complete HTTP response, so it cannot parse a stream that never ends. The script imports the community `k6/x/sse` extension -(`github.com/phymbert/xk6-sse`). Newer k6 releases resolve community extensions -automatically. If the stock `grafana/k6` image rejects the import, build a -binary with the extension: +(`github.com/phymbert/xk6-sse`), which a stock k6 binary does not include. + +Build the pinned binary once: ```bash make k6-build # creates k6/k6-sse (git-ignored) -K6=./k6/k6-sse make load +./k6/k6-sse version # expect k6 v2.0.0 + xk6-sse v0.2.0 ``` -`make load` uses, in order: `k6` on `PATH`, `k6/k6-sse`, then -`docker run --rm -i grafana/k6`. +The build pins **k6 v2.0.0**. On k6 v2.4.0 the same `xk6-sse` v0.2.0 build +fails at runtime with `TypeError: Object has no member 'on'` (the extension's +client methods are not exposed); v2.0.0 was verified locally on 2026-11-14. If +you bump `K6_BUILD_VERSION`, rerun the script against a test stream first. + +`make load` uses, in order: `K6` if set, `k6/k6-sse`, then `k6` on `PATH`. +A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension. ## Procedure @@ -35,7 +40,7 @@ K6=./k6/k6-sse make load ```bash make load - # or: API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js + # or: API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js ``` `make load` writes `k6/results/summary-.json` (git-ignored). @@ -56,25 +61,35 @@ K6=./k6/k6-sse make load 4. Record these numbers in the paper/README from the saved summary: - `sse_connections_opened` count (must be 120) - - `sse_connections_active` max (must reach 120) + - `sse_connections_closed` count (must equal connections opened) - `sse_open_success` rate (must be >= 99%) - `sse_event_latency` p95 (target < 2000 ms) - `sse_connection_errors` (target 0) - `sse_events_received` total +## Local validation before the cluster run + +The script was validated locally on 2026-11-14 with the pinned k6 v2.0.0 build +against a mock SSE stream and the real API image (3 VUs, 1 heartbeat each): +connections opened and closed cleanly, open success 100%. The real 120-client +run still requires the deployed API, because localhost results say nothing +about Cloudflare buffering or SKE scheduling. + ## Pass criteria | Metric | Target | |---|---| | Connections opened | 120 | -| Peak active connections | 120 | +| Connections closed (stream ended) | 120 | | Open success rate | >= 99% | | p95 event latency | < 2000 ms | | Unexpected connection errors | 0 | | Duration | 5 minutes | -The k6 thresholds enforce the first, third, and fourth; the others are read -from the summary. A p95 above 2 s means Cloudflare, the Go SSE hub, or the +The k6 thresholds enforce connections opened, open success, and p95 latency; +the others are read from the summary. A final `sse_connections_closed` below +120 means at least one opened VU did not finish; inspect +`sse_connection_errors` for unexpected errors. A p95 above 2 s means Cloudflare, the Go SSE hub, or the client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s heartbeat, and whether the connection is HTTP/2-buffered at the edge. @@ -83,6 +98,10 @@ heartbeat, and whether the connection is HTTP/2-buffered at the edge. - `sse_event_latency` uses the server timestamp in `data.t` (ping) or `data.at` (scenario). It includes clock skew between the k6 host and the pod; keep both on NTP. Negative or > 1 h values are ignored. +- The initial `scenario` event on connect replays the current state and carries + the original state-change time, not the send time. The script excludes that + first scenario event per VU; only scenario changes broadcast during the run + are sampled by `sse_scenario_latency`. Heartbeat pings always count. - Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends cleanly instead of waiting for the 7-minute request timeout. - One API replica means all clients share one SSE hub; a pod restart drops diff --git a/k6/sse.js b/k6/sse.js index af0f09e..3aefcc4 100644 --- a/k6/sse.js +++ b/k6/sse.js @@ -2,17 +2,17 @@ // // The native k6 http module buffers the whole response, so it cannot read // Server-Sent Events incrementally. This script uses the community -// `k6/x/sse` extension (github.com/phymbert/xk6-sse). Recent k6 releases -// resolve `k6/x/sse` automatically; otherwise build a binary with: +// `k6/x/sse` extension (github.com/phymbert/xk6-sse), which is not in the +// stock k6 binary. Build the pinned binary once: // make k6-build // // Run: -// API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js +// API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js // make load // // Recorded metrics: // sse_connections_opened counter, expected >= SSE_VUS (120) -// sse_connections_active gauge, peak should reach 120 +// sse_connections_closed counter, expected to equal connections_opened // sse_open_success rate of VUs that opened a stream // sse_event_latency p95 server-timestamp-to-client latency, ms // sse_scenario_latency same, scenario events only (if any occur) @@ -24,7 +24,7 @@ import http from 'k6/http'; import sse from 'k6/x/sse'; import { check } from 'k6'; -import { Counter, Gauge, Rate, Trend } from 'k6/metrics'; +import { Counter, Rate, Trend } from 'k6/metrics'; const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, ''); const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`; @@ -33,7 +33,7 @@ const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20)); const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m'; const connectionsOpened = new Counter('sse_connections_opened'); -const connectionsActive = new Gauge('sse_connections_active'); +const connectionsClosed = new Counter('sse_connections_closed'); const openSuccess = new Rate('sse_open_success'); const connectionErrors = new Counter('sse_connection_errors'); const eventsReceived = new Counter('sse_events_received'); @@ -73,23 +73,10 @@ export function setup() { export default function () { let opened = false; - let active = false; let closedByUs = false; + let firstScenario = true; let pings = 0; - const markActive = () => { - if (!active) { - active = true; - connectionsActive.add(1); - } - }; - const markInactive = () => { - if (active) { - active = false; - connectionsActive.add(-1); - } - }; - const params = { method: 'GET', headers: { @@ -104,7 +91,6 @@ export default function () { const res = sse.open(STREAM_URL, params, function (client) { client.on('open', function () { opened = true; - markActive(); connectionsOpened.add(1); }); @@ -123,9 +109,18 @@ export default function () { } // The contract puts the emit time in `t` (ping) or `at` (scenario). + // The first scenario event on connect replays the current state and + // carries the original state-change timestamp, not the send time, so + // exclude it per VU. Only scenario changes broadcast during the run + // are valid samples. + const initialScenario = name === 'scenario' && firstScenario; + if (name === 'scenario') { + firstScenario = false; + } + // Latency is server timestamp -> client receipt, in milliseconds. const stamp = payload && (payload.t || payload.at); - if (stamp) { + if (stamp && !initialScenario) { const emitted = Date.parse(stamp); if (!Number.isNaN(emitted)) { const lag = now - emitted; @@ -142,7 +137,6 @@ export default function () { pings += 1; if (pings >= HOLD_PINGS && !closedByUs) { closedByUs = true; - markInactive(); client.close(); } } @@ -153,10 +147,12 @@ export default function () { connectionErrors.add(1); console.error(`SSE error: ${e && e.error ? e.error() : e}`); } - markInactive(); }); }); + if (opened) { + connectionsClosed.add(1); + } check(res, { 'sse status 200': (r) => r && r.status === 200, 'sse content-type': (r) => @@ -168,5 +164,4 @@ export default function () { connectionErrors.add(1); } openSuccess.add(opened); - markInactive(); } diff --git a/scripts/gate1-check.sh b/scripts/gate1-check.sh new file mode 100755 index 0000000..9f444fe --- /dev/null +++ b/scripts/gate1-check.sh @@ -0,0 +1,470 @@ +#!/usr/bin/env bash +# Read-only Gate 1 readiness check. +# +# It NEVER applies, creates, patches, deletes, scales, or restarts anything. +# Allowed calls: kubectl config/get/auth can-i/apply --dry-run=client, curl. +# +# Usage (normally through make): +# export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml +# make gate1-check +# make gate1-check CHECK_PUBLIC=1 # also curl the public URLs +# make gate1-check ALLOW_PENDING=1 # exit 0 while human secrets are pending +# +# Exit codes: 0 ready, 1 hard failure (permissions/manifests/cluster), 2 only +# human prerequisites are missing (Secrets, Deployments, public URLs). +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +NS="${NS:-fch26-onemanarmy}" +CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}" +REGISTRY="${REGISTRY:-git.noelsabia.dev}" +OWNER="${OWNER:-noelsabia}" +TAG="${TAG:-gate1}" +CHECK_PUBLIC="${CHECK_PUBLIC:-0}" +ALLOW_PENDING="${ALLOW_PENDING:-0}" +WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}" +API_URL="${API_URL:-https://api.pulse.noelsabia.dev}" +CURL_MAX_TIME="${CURL_MAX_TIME:-15}" + +hard=0 +pending=0 +warnings=0 + +ok() { printf '[ok] %s\n' "$*"; } +bad() { printf '[FAIL] %s\n' "$*"; hard=$((hard + 1)); } +missing() { printf '[pending] %s\n' "$*"; pending=$((pending + 1)); } +warn() { printf '[warn] %s\n' "$*"; warnings=$((warnings + 1)); } +section() { printf '\n== %s\n' "$*"; } + +finish() { + printf '\n== Gate 1 readiness summary\n' + printf 'hard failures: %d\n' "$hard" + printf 'pending prerequisites: %d\n' "$pending" + printf 'warnings: %d\n' "$warnings" + if ((hard > 0)); then + printf 'RESULT: NOT READY (fix the failures above)\n' + exit 1 + fi + if ((pending > 0)); then + if [[ "$ALLOW_PENDING" == "1" ]]; then + printf 'RESULT: READY except for pending prerequisites (ALLOW_PENDING=1)\n' + exit 0 + fi + printf 'RESULT: NOT READY (prerequisites above are still pending)\n' + exit 2 + fi + printf 'RESULT: READY for Gate 1\n' + exit 0 +} + +trap finish EXIT + +printf 'Gate 1 readiness (read-only; no cluster mutation)\n' +printf 'namespace=%s context=%s tag=%s\n' "$NS" "$CONTEXT" "$TAG" + +section "kubeconfig and cluster" +if [[ -z "${KUBECONFIG:-}" ]]; then + bad "KUBECONFIG is not set" + printf ' export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml\n' +else + if [[ ! -f "$KUBECONFIG" ]]; then + bad "KUBECONFIG does not point to a file: $KUBECONFIG" + else + mode="$(stat -c '%a' "$KUBECONFIG" 2>/dev/null || echo unknown)" + if [[ "$mode" != "600" && "$mode" != "400" ]]; then + warn "kubeconfig mode is $mode; run: chmod 600 \"$KUBECONFIG\"" + else + ok "kubeconfig mode $mode" + fi + fi +fi + +if ! command -v kubectl >/dev/null 2>&1; then + bad "kubectl not found" + exit 1 +fi + +if ! kubectl config get-contexts -o name 2>/dev/null | grep -Fxq "$CONTEXT"; then + bad "context '$CONTEXT' not found in KUBECONFIG" + exit 1 +fi +ok "context '$CONTEXT' exists" + +KC=(kubectl --context "$CONTEXT" --namespace "$NS") + +if "${KC[@]}" get --raw=/readyz --request-timeout=5s >/dev/null 2>&1; then + ok "API server reachable" +else + bad "API server is not reachable with context '$CONTEXT'" + exit 1 +fi + +if "${KC[@]}" get deployments >/dev/null 2>&1; then + ok "namespace '$NS' is readable" +else + bad "cannot read namespace '$NS'" + exit 1 +fi + +section "namespace permissions (expected values)" +check_can_i() { + local expected="$1" verb="$2" resource="$3" got + got="$("${KC[@]}" auth can-i "$verb" "$resource" 2>/dev/null)" + if [[ "$got" == "$expected" ]]; then + ok "$verb $resource -> $got" + else + bad "$verb $resource -> ${got:-} (expected $expected)" + fi +} +check_can_i yes create deployments +check_can_i yes create services +check_can_i yes create configmaps +check_can_i yes create secrets +check_can_i yes create pods +check_can_i no create ingresses +check_can_i no create jobs +check_can_i no create cronjobs +check_can_i no create networkpolicies +check_can_i no create roles +check_can_i no create rolebindings + +section "manifest guardrails" +if manifest_out="$(python3 - "$ROOT/deploy/k8s" "$NS" "$TAG" "$REGISTRY" "$OWNER" <<'PY' 2>&1 +import glob +import os +import re +import sys + +root, ns, tag, registry, owner = sys.argv[1:6] +errors = [] +warns = [] +docs = [] + +for path in sorted(glob.glob(os.path.join(root, "*.yaml"))): + with open(path, encoding="utf-8") as handle: + try: + loaded = [d for d in __import__("yaml").safe_load_all(handle) if d] + except Exception as exc: # noqa: BLE001 - report the parse error verbatim + errors.append(f"{os.path.basename(path)}: YAML parse error: {exc}") + continue + if not loaded: + errors.append(f"{os.path.basename(path)}: no YAML documents") + for doc in loaded: + doc["__file__"] = os.path.basename(path) + docs.append(doc) + +if not docs: + errors.append("deploy/k8s contains no manifests") + +allowed = {"ConfigMap", "Deployment", "Service"} +forbidden = { + "Ingress", "Job", "CronJob", "NetworkPolicy", "Role", "RoleBinding", + "ClusterRole", "ClusterRoleBinding", "CustomResourceDefinition", + "Namespace", "PersistentVolume", "PersistentVolumeClaim", "StorageClass", + "PodSecurityPolicy", "PriorityClass", "APIService", + "MutatingWebhookConfiguration", "ValidatingWebhookConfiguration", +} + +objects = {} +configmaps = {} +services = {} +deployments = {} + +for doc in docs: + where = f"{doc['__file__']}:{doc.get('kind', '?')}/{doc.get('metadata', {}).get('name', '?')}" + kind = doc.get("kind") + meta = doc.get("metadata") or {} + name = meta.get("name") + if not doc.get("apiVersion") or not kind or not name: + errors.append(f"{where}: missing apiVersion, kind, or metadata.name") + continue + if kind not in allowed: + errors.append(f"{where}: kind {kind} is not allowed (namespace-only: ConfigMap, Deployment, Service)") + continue + if kind in forbidden: + errors.append(f"{where}: forbidden kind {kind}") + if meta.get("namespace") != ns: + errors.append(f"{where}: metadata.namespace is {meta.get('namespace')!r}, expected {ns!r}") + key = (kind, name) + if key in objects: + errors.append(f"{where}: duplicate object") + objects[key] = doc + if kind == "ConfigMap": + configmaps[name] = doc + elif kind == "Service": + services[name] = doc + elif kind == "Deployment": + deployments[name] = doc + +for name, cm in configmaps.items(): + data = cm.get("data") or {} + for key in data: + if re.search(r"(?i)(token|password|secret|private[_-]?key|credential)", key): + errors.append(f"ConfigMap/{name}: key {key!r} looks like a secret; use a Secret created by the human") + +required_config = { + "pulsecast-shared": {"PUBLIC_WEB_URL", "PUBLIC_API_URL", "API_INTERNAL_URL", "ML_INTERNAL_URL"}, + "web-config": {"PORT", "NUXT_PUBLIC_API_BASE", "NUXT_API_INTERNAL_BASE"}, + "api-config": {"PORT", "DATA_DIR", "FASTAPI_URL", "ML_TIMEOUT", "SSE_HEARTBEAT"}, + "ml-config": { + "PORT", "PULSECAST_ROOT", "PULSECAST_ARTIFACTS", "PULSECAST_WEATHER", + "PULSECAST_MODELS", "PULSECAST_UPLOADS", + }, +} +for name, keys in required_config.items(): + cm = configmaps.get(name) + if cm is None: + errors.append(f"missing ConfigMap/{name}") + continue + have = set((cm.get("data") or {}).keys()) + for key in sorted(keys - have): + errors.append(f"ConfigMap/{name}: missing key {key}") + +expected_ports = {"web": 3000, "api": 8080, "ml": 8000, "cloudflared": 2000} +expected_uids = {"web": 10001, "api": 65532, "ml": 10001, "cloudflared": 65532} +expected_images = { + "web": f"{registry}/{owner}/pulsecast-web:{tag}", + "api": f"{registry}/{owner}/pulsecast-api:{tag}", + "ml": f"{registry}/{owner}/pulsecast-ml:{tag}", +} +gitea_images = {"web", "api", "ml"} + +for name in ("web", "api", "ml", "cloudflared"): + dep = deployments.get(name) + if dep is None: + errors.append(f"missing Deployment/{name}") + continue + spec = dep.get("spec") or {} + if spec.get("replicas") != 1: + errors.append(f"Deployment/{name}: replicas should be 1") + template = (spec.get("template") or {}) + pod = template.get("spec") or {} + if pod.get("automountServiceAccountToken") is not False: + errors.append(f"Deployment/{name}: automountServiceAccountToken must be false") + pod_security = pod.get("securityContext") or {} + if (pod_security.get("seccompProfile") or {}).get("type") != "RuntimeDefault": + errors.append(f"Deployment/{name}: seccompProfile must be RuntimeDefault") + for field in ("runAsUser", "runAsGroup"): + value = pod_security.get(field) + if not isinstance(value, int) or value <= 0: + errors.append(f"Deployment/{name}: {field} must be a positive numeric id (SKE runAsNonRoot caveat)") + elif value != expected_uids[name]: + errors.append(f"Deployment/{name}: {field} is {value}, expected {expected_uids[name]}") + if pod_security.get("runAsNonRoot") is True and pod_security.get("runAsUser") == 0: + errors.append(f"Deployment/{name}: runAsUser 0 conflicts with runAsNonRoot") + for field in ("hostNetwork", "hostPID", "hostIPC"): + if pod.get(field): + errors.append(f"Deployment/{name}: {field} is not allowed") + if name in gitea_images: + pulls = pod.get("imagePullSecrets") or [] + if not any(p.get("name") == "gitea-regcred" for p in pulls): + errors.append(f"Deployment/{name}: imagePullSecrets must include gitea-regcred") + containers = pod.get("containers") or [] + if len(containers) != 1: + errors.append(f"Deployment/{name}: expected exactly one container") + continue + container = containers[0] + if container.get("name") != name: + errors.append(f"Deployment/{name}: container name is {container.get('name')!r}") + image = container.get("image") or "" + if image.endswith(":latest"): + errors.append(f"Deployment/{name}: image tag must be pinned, not :latest") + if name in gitea_images and not image.startswith(f"{registry}/{owner}/"): + errors.append(f"Deployment/{name}: image {image!r} is not under {registry}/{owner}/") + if name in expected_images and image != expected_images[name]: + warns.append(f"Deployment/{name}: image {image!r} does not match TAG={tag}") + if name == "cloudflared" and not image.startswith("cloudflare/cloudflared:"): + errors.append(f"Deployment/{name}: unexpected image {image!r}") + if name in expected_ports: + ports = container.get("ports") or [] + if not any(int(p.get("containerPort", -1)) == expected_ports[name] for p in ports): + errors.append(f"Deployment/{name}: containerPort {expected_ports[name]} not found") + for port in ports: + if port.get("hostPort"): + errors.append(f"Deployment/{name}: hostPort is not allowed") + security = container.get("securityContext") or {} + if security.get("runAsNonRoot") is not True: + errors.append(f"Deployment/{name}: runAsNonRoot must be true") + if security.get("allowPrivilegeEscalation") is not False: + errors.append(f"Deployment/{name}: allowPrivilegeEscalation must be false") + if security.get("readOnlyRootFilesystem") is not True: + errors.append(f"Deployment/{name}: readOnlyRootFilesystem must be true") + if security.get("privileged") is True: + errors.append(f"Deployment/{name}: privileged is not allowed") + if security.get("runAsUser") == 0: + errors.append(f"Deployment/{name}: runAsUser 0 is not allowed") + drops = ((security.get("capabilities") or {}).get("drop") or []) + if "ALL" not in drops: + errors.append(f"Deployment/{name}: capabilities.drop must include ALL") + resources = container.get("resources") or {} + requests = resources.get("requests") or {} + limits = resources.get("limits") or {} + for group_name, group in (("requests", requests), ("limits", limits)): + for key in ("cpu", "memory"): + if not group.get(key): + errors.append(f"Deployment/{name}: resources.{group_name}.{key} is missing") + mounts = container.get("volumeMounts") or [] + tmp_mounts = [m for m in mounts if m.get("mountPath") == "/tmp"] + if not tmp_mounts: + errors.append(f"Deployment/{name}: /tmp volumeMount is missing") + else: + volume_name = tmp_mounts[0].get("name") + volumes = pod.get("volumes") or [] + if not any(v.get("name") == volume_name and "emptyDir" in v for v in volumes): + errors.append(f"Deployment/{name}: /tmp must be backed by an emptyDir") + for probe in ("startupProbe", "livenessProbe", "readinessProbe"): + probe_spec = container.get(probe) or {} + http = probe_spec.get("httpGet") or {} + if not http.get("path") or not http.get("port"): + errors.append(f"Deployment/{name}: {probe}.httpGet is missing") + refs = [] + for source in container.get("envFrom") or []: + if source.get("configMapRef"): + refs.append(source["configMapRef"].get("name")) + for env in container.get("env") or []: + if env.get("valueFrom", {}).get("configMapKeyRef"): + refs.append(env["valueFrom"]["configMapKeyRef"].get("name")) + for ref in refs: + if ref not in configmaps: + errors.append(f"Deployment/{name}: references missing ConfigMap/{ref}") + expected_secret_refs = { + "api": { + ("control-auth", "CONTROL_USER"), + ("control-auth", "CONTROL_PASS"), + ("control-auth", "CONTROL_TOKEN"), + }, + "ml": {("control-auth", "CONTROL_TOKEN")}, + "cloudflared": {("cloudflared", "TUNNEL_TOKEN")}, + } + seen_secret_refs = set() + for env in container.get("env") or []: + secret_ref = env.get("valueFrom", {}).get("secretKeyRef") + if not secret_ref: + continue + secret_name = secret_ref.get("name") + secret_key = secret_ref.get("key") + if secret_name not in ("gitea-regcred", "cloudflared", "control-auth"): + errors.append(f"Deployment/{name}: unexpected Secret reference {secret_name!r}") + seen_secret_refs.add((secret_name, secret_key)) + for required in expected_secret_refs.get(name, set()): + if required not in seen_secret_refs: + errors.append(f"Deployment/{name}: missing Secret reference {required[0]}/{required[1]}") + +if "cloudflared" in deployments: + args = ((deployments["cloudflared"].get("spec") or {}).get("template") or {}).get("spec", {}).get("containers", [{}])[0].get("args") or [] + for required in ("tunnel", "run", "--token", "$(TUNNEL_TOKEN)", "--no-autoupdate", "--protocol", "http2"): + if required not in args: + errors.append(f"Deployment/cloudflared: args must include {required!r}") + +for name in ("web", "api", "ml"): + svc = services.get(name) + if svc is None: + errors.append(f"missing Service/{name}") + continue + spec = svc.get("spec") or {} + if spec.get("type") != "ClusterIP": + errors.append(f"Service/{name}: type must be ClusterIP") + for field in ("externalIPs", "loadBalancerIP", "loadBalancerSourceRanges"): + if spec.get(field): + errors.append(f"Service/{name}: {field} is not allowed") + ports = spec.get("ports") or [] + if not any(int(p.get("port", -1)) == expected_ports[name] for p in ports): + errors.append(f"Service/{name}: port {expected_ports[name]} not found") + if any(p.get("nodePort") for p in ports): + errors.append(f"Service/{name}: nodePort is not allowed") + +kinds = {} +for kind, _ in objects: + kinds[kind] = kinds.get(kind, 0) + 1 +summary = ", ".join(f"{kind}={count}" for kind, count in sorted(kinds.items())) +print(f"parsed {len(objects)} objects ({summary})") +for message in warns: + print(f"WARN: {message}") +for message in errors: + print(f"FAIL: {message}") +sys.exit(1 if errors else 0) +PY +)"; then + ok "static guardrails passed" + printf '%s\n' "$manifest_out" | sed 's/^/ /' +else + bad "static guardrails failed" + printf '%s\n' "$manifest_out" | sed 's/^/ /' +fi + +section "kubectl client dry-run" +if dry_run_out="$("${KC[@]}" apply --dry-run=client -f "$ROOT/deploy/k8s" 2>&1)"; then + count="$(printf '%s\n' "$dry_run_out" | grep -c '(dry run)')" + ok "kubectl apply --dry-run=client passed ($count resources)" +else + bad "kubectl apply --dry-run=client failed" + printf '%s\n' "$dry_run_out" | sed 's/^/ /' +fi + +section "human prerequisites: Secrets" +check_secret_keys() { + local secret="$1" + shift + if ! "${KC[@]}" get secret "$secret" >/dev/null 2>&1; then + missing "Secret/$secret is missing (create it with docs/deploy.md; never commit it)" + return + fi + ok "Secret/$secret exists" + local have key + have="$("${KC[@]}" get secret "$secret" -o go-template='{{range $k,$v := .data}}{{$k}}{{"\n"}}{{end}}' 2>/dev/null || true)" + for key in "$@"; do + if printf '%s\n' "$have" | grep -qx "$key"; then + ok "Secret/$secret has key $key" + else + missing "Secret/$secret is missing key $key" + fi + done +} + +check_secret_keys gitea-regcred .dockerconfigjson +if "${KC[@]}" get secret gitea-regcred >/dev/null 2>&1; then + secret_type="$("${KC[@]}" get secret gitea-regcred -o jsonpath='{.type}' 2>/dev/null || true)" + if [[ "$secret_type" == "kubernetes.io/dockerconfigjson" ]]; then + ok "Secret/gitea-regcred type is kubernetes.io/dockerconfigjson" + else + bad "Secret/gitea-regcred type is ${secret_type:-}, expected kubernetes.io/dockerconfigjson" + fi +fi +check_secret_keys cloudflared TUNNEL_TOKEN +check_secret_keys control-auth CONTROL_USER CONTROL_PASS CONTROL_TOKEN + +section "deployment status" +for deployment in web api ml cloudflared; do + if ! "${KC[@]}" get deployment "$deployment" >/dev/null 2>&1; then + missing "Deployment/$deployment is not deployed yet" + continue + fi + ready="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)" + desired="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" + image="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true)" + if [[ -n "$ready" && -n "$desired" ]] && ((ready >= desired)); then + ok "Deployment/$deployment ready ($ready/$desired, $image)" + else + bad "Deployment/$deployment not ready (${ready:-0}/${desired:-?}, $image)" + fi + if [[ "$deployment" != "cloudflared" && "$image" != *":$TAG" ]]; then + warn "Deployment/$deployment image tag is not :$TAG" + fi +done + +if [[ "$CHECK_PUBLIC" == "1" ]]; then + section "public HTTPS" + for target in "web|$WEB_URL/" "api|$API_URL/api/healthz"; do + name="${target%%|*}" + url="${target#*|}" + code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "$CURL_MAX_TIME" "$url" 2>/dev/null || echo 000)" + if [[ "$code" == "200" ]]; then + ok "$name public URL -> HTTP 200 ($url)" + else + missing "$name public URL -> HTTP ${code:-000} ($url)" + fi + done +else + section "public HTTPS" + printf '[skip] CHECK_PUBLIC=0; run make smoke (or gate1-check CHECK_PUBLIC=1) after deploy\n' +fi diff --git a/scripts/smoke.sh b/scripts/smoke.sh old mode 100644 new mode 100755