feat: harden Gate 1 manifests and runbook
- add read-only make gate1-check: permissions, guardrails, dry-run, Secret keys - set numeric runAsUser/runAsGroup (SKE non-numeric user caveat), pin cloudflared and use HTTP/2 - align ML env with PULSECAST_* and wire CONTROL_TOKEN for API/ML training proxy - fix service worktree paths and smoke.sh executable bit - pin k6 v2.0.0 for xk6-sse, fix closed-connection metric and initial scenario latency - rewrite docs/deploy.md as an exact copy-paste Gate 1 checklist
This commit is contained in:
1 parent
55dfe8b616
commit
5b199f8fe3
13 files changed
+1231
-235
No files matched your search
@@ -16,13 +16,15 @@ WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
|
||||
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
|
||||
ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG)
|
||||
|
||||
# Service checkouts live beside the main checkout or in per-task worktrees:
|
||||
# Hackathon_2026/worktrees/<repo>/<branch>
|
||||
# Service repos are bare Git repos beside this one, each with per-task worktrees:
|
||||
# Hackathon_2026/web-nuxt.git/feat-web-bootstrap
|
||||
# Hackathon_2026/api-go.git/feat-api-bootstrap
|
||||
# Hackathon_2026/ML-py.git/feat-ml-bootstrap
|
||||
# Override any *_DIR on the command line when the layout differs.
|
||||
SERVICES_ROOT ?= $(if $(wildcard $(abspath $(ROOT)/../../..)/worktrees),$(abspath $(ROOT)/../../..),$(abspath $(ROOT)/..))
|
||||
WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/web-nuxt/*) $(wildcard $(SERVICES_ROOT)/web-nuxt)),$(SERVICES_ROOT)/web-nuxt)
|
||||
API_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/api-go/*) $(wildcard $(SERVICES_ROOT)/api-go)),$(SERVICES_ROOT)/api-go)
|
||||
ML_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/ML-py/*) $(wildcard $(SERVICES_ROOT)/ML-py)),$(SERVICES_ROOT)/ML-py)
|
||||
SERVICE_REPOS ?= $(abspath $(ROOT)/../..)
|
||||
WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/web-nuxt.git/main)))
|
||||
API_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/main)))
|
||||
ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main)))
|
||||
|
||||
WEB_URL ?= https://pulse.noelsabia.dev
|
||||
API_URL ?= https://api.pulse.noelsabia.dev
|
||||
@@ -36,10 +38,13 @@ SSE_HOLD_PINGS ?= 20
|
||||
SSE_MAX_DURATION ?= 6m
|
||||
K6 ?=
|
||||
K6_BIN ?= $(ROOT)/k6/k6-sse
|
||||
# xk6-sse v0.2.0 exposes client.on/client.close correctly on k6 v2.0.x but not
|
||||
# on v2.4.0 (verified 2026-11-14). Pin the known-good k6 version.
|
||||
K6_BUILD_VERSION ?= v2.0.0
|
||||
K6_SCRIPT := $(ROOT)/k6/sse.js
|
||||
K6_RESULTS := $(ROOT)/k6/results
|
||||
|
||||
.PHONY: help validate validate-kubeconform build build-web build-api build-ml push push-web push-api push-ml set-image deploy deploy-check status logs smoke smoke-local load k6-build secret-commands clean
|
||||
.PHONY: help validate validate-kubeconform gate1-check build build-web build-api build-ml push push-web push-api push-ml set-image restart deploy deploy-check status logs smoke smoke-local load k6-build secret-commands clean
|
||||
|
||||
help: ## Show available targets
|
||||
@awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
||||
@@ -49,9 +54,9 @@ validate: ## Validate deploy/k8s read-only (schema when possible, never applies)
|
||||
if command -v "$(KUBECONFORM)" >/dev/null 2>&1; then \
|
||||
echo "== kubeconform (strict, offline schemas)"; \
|
||||
"$(KUBECONFORM)" -strict -summary -ignore-missing-schemas "$(ROOT)/deploy/k8s"; \
|
||||
elif command -v kubectl >/dev/null 2>&1 && kubectl cluster-info >/dev/null 2>&1; then \
|
||||
elif command -v kubectl >/dev/null 2>&1 && kubectl --context "$(CONTEXT)" get --raw=/readyz --request-timeout=5s >/dev/null 2>&1; then \
|
||||
echo "== kubectl client dry-run (read-only)"; \
|
||||
kubectl apply --dry-run=client -f "$(ROOT)/deploy/k8s"; \
|
||||
$(KUBECTL) apply --dry-run=client -f "$(ROOT)/deploy/k8s"; \
|
||||
else \
|
||||
echo "== YAML syntax only: kubeconform is not installed and no cluster is reachable"; \
|
||||
"$(PYTHON)" -c "import glob, yaml; docs=[d for f in sorted(glob.glob('$(ROOT)/deploy/k8s/*.yaml')) for d in yaml.safe_load_all(open(f)) if d]; assert all(isinstance(d, dict) and d.get('apiVersion') and d.get('kind') for d in docs), 'manifest missing apiVersion/kind'; print(f'{len(docs)} manifests parsed')"; \
|
||||
@@ -62,6 +67,10 @@ validate-kubeconform: ## Validate schemas with kubeconform (must be installed)
|
||||
@command -v "$(KUBECONFORM)" >/dev/null 2>&1 || { echo "kubeconform not found; go install github.com/yannh/kubeconform/cmd/kubeconform@latest"; exit 1; }
|
||||
"$(KUBECONFORM)" -strict -summary -ignore-missing-schemas "$(ROOT)/deploy/k8s"
|
||||
|
||||
gate1-check: ## Read-only Gate 1 readiness: permissions, guardrails, dry-run, Secrets (never mutates)
|
||||
@NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" REGISTRY="$(REGISTRY)" OWNER="$(OWNER)" TAG="$(TAG)" \
|
||||
"$(ROOT)/scripts/gate1-check.sh"
|
||||
|
||||
build: build-web build-api build-ml ## Build all three images locally
|
||||
|
||||
build-web: ## Build the web image
|
||||
@@ -97,12 +106,22 @@ set-image: ## Update live Deployments to TAG (guarded; human only)
|
||||
$(KUBECTL) set image deployment/ml ml="$(ML_IMAGE)"
|
||||
$(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s
|
||||
|
||||
deploy-check: ## Verify required Secrets exist (read-only)
|
||||
restart: ## Force a re-pull/restart of web, api, ml after re-pushing the same tag (guarded)
|
||||
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
|
||||
$(KUBECTL) rollout restart deployment/web deployment/api deployment/ml
|
||||
$(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s
|
||||
|
||||
deploy-check: ## Verify required Secrets and keys exist (read-only)
|
||||
@for s in gitea-regcred cloudflared control-auth; do \
|
||||
$(KUBECTL) get secret "$$s" >/dev/null 2>&1 || { \
|
||||
echo "Missing Secret $$s in $(NAMESPACE); see docs/deploy.md"; exit 1; }; \
|
||||
done
|
||||
@echo "required Secrets present"
|
||||
@for ref in cloudflared:TUNNEL_TOKEN control-auth:CONTROL_USER control-auth:CONTROL_PASS control-auth:CONTROL_TOKEN; do \
|
||||
secret="$${ref%%:*}"; key="$${ref#*:}"; \
|
||||
value="$$($(KUBECTL) get secret "$$secret" -o jsonpath="{.data.$$key}" 2>/dev/null || true)"; \
|
||||
test -n "$$value" || { echo "Secret $$secret is missing key $$key; see docs/deploy.md"; exit 1; }; \
|
||||
done
|
||||
@echo "required Secrets and keys present"
|
||||
|
||||
deploy: ## Apply manifests (guarded; human only)
|
||||
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
|
||||
@@ -154,8 +173,11 @@ load: ## Run the 120-client / 5-minute SSE load test
|
||||
k6-build: ## Build a k6 binary with the SSE extension (k6/k6-sse)
|
||||
@command -v go >/dev/null 2>&1 || { echo "Go is required to build xk6"; exit 1; }
|
||||
@cd "$(ROOT)/k6" && GOBIN="$(ROOT)/k6" go install go.k6.io/xk6/cmd/xk6@latest
|
||||
@cd "$(ROOT)/k6" && "$(ROOT)/k6/xk6" build --with github.com/phymbert/xk6-sse@latest --output "$(ROOT)/k6/k6-sse"
|
||||
@echo "Built $(ROOT)/k6/k6-sse"
|
||||
@cd "$(ROOT)/k6" && "$(ROOT)/k6/xk6" build \
|
||||
--k6-version "$(K6_BUILD_VERSION)" \
|
||||
--with github.com/phymbert/xk6-sse@latest \
|
||||
--output "$(ROOT)/k6/k6-sse"
|
||||
@echo "Built $(ROOT)/k6/k6-sse (k6 $(K6_BUILD_VERSION) + xk6-sse)"
|
||||
|
||||
secret-commands: ## Print the exact Secret creation commands (creates nothing)
|
||||
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic cloudflared \\"
|
||||
@@ -163,7 +185,8 @@ secret-commands: ## Print the exact Secret creation commands (creates nothing)
|
||||
@echo
|
||||
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic control-auth \\"
|
||||
@echo " --from-literal=CONTROL_USER='<control user>' \\"
|
||||
@echo " --from-literal=CONTROL_PASS='<strong password>'"
|
||||
@echo " --from-literal=CONTROL_PASS='<strong password>' \\"
|
||||
@echo " --from-literal=CONTROL_TOKEN='<random token shared by API and ML>'"
|
||||
@echo
|
||||
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret docker-registry gitea-regcred \\"
|
||||
@echo " --docker-server=$(REGISTRY) \\"
|
||||
|
||||
@@ -10,6 +10,7 @@ commands in `docs/deploy.md`.
|
||||
deploy/k8s/ web, api, ml + cloudflared Deployments and Services, ConfigMaps
|
||||
deploy/examples/ example Secret shapes (placeholders only, never applied)
|
||||
scripts/smoke.sh public + in-cluster smoke checks
|
||||
scripts/gate1-check.sh read-only permissions/manifest/secret readiness check
|
||||
k6/sse.js 120 concurrent SSE clients for 5 minutes
|
||||
docs/deploy.md Gate 1 runbook and exact human checklist
|
||||
docs/load-test.md load-test procedure and how to record results
|
||||
@@ -21,16 +22,34 @@ Makefile validate, build, push, deploy, smoke, load helpers
|
||||
|
||||
```bash
|
||||
export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml # chmod 600
|
||||
make gate1-check # read-only: permissions, guardrails, dry-run, Secrets
|
||||
make validate # client-side dry run only, never applies
|
||||
make build # all three images
|
||||
make push # after docker login to Gitea
|
||||
make deploy CONFIRM=yes # human only; requires the three Secrets first
|
||||
make smoke # public HTTPS + API + ML health checks
|
||||
make load # k6 SSE load test (needs the k6 SSE extension)
|
||||
make k6-build # once: pinned k6 + SSE extension
|
||||
make load # 120 SSE clients for 5 minutes
|
||||
```
|
||||
|
||||
`make gate1-check` exits 0 when ready, 1 on a hard permission/manifest failure,
|
||||
and 2 when only human prerequisites (Secrets, Deployments) are pending. It
|
||||
never mutates the cluster. DNS and public HTTPS are checked separately in the
|
||||
runbook and by `make smoke`.
|
||||
|
||||
Read `docs/deploy.md` before the first deploy. It contains the Gitea token,
|
||||
`docker login`, hello-world push, `gitea-regcred`, Cloudflare Tunnel, public
|
||||
hostname, and verification steps.
|
||||
|
||||
## Gate 1 status
|
||||
|
||||
- Manifests validate read-only; `make gate1-check` is green except for pending
|
||||
human prerequisites.
|
||||
- All three images build and pass local manifest-equivalent runtime checks
|
||||
(read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker.
|
||||
- Still human-only: Gitea read-only token + hello image pull, `gitea-regcred`,
|
||||
Cloudflare Tunnel + DNS hostnames, and the three Secrets.
|
||||
|
||||
## Git workflow
|
||||
|
||||
`main` is integration. New work happens on a `feat/<topic>` branch in its own
|
||||
@@ -46,3 +65,8 @@ committed directly to `main`.
|
||||
- No secrets, tokens, kubeconfig, or raw CSV in git.
|
||||
- The human applies cluster mutations. `make deploy` refuses to run without
|
||||
`CONFIRM=yes`.
|
||||
- Every pod sets an explicit numeric `runAsUser`/`runAsGroup`: web/ml
|
||||
`10001:10001`, api/cloudflared `65532:65532`. SKE rejects a non-numeric image
|
||||
user under `runAsNonRoot`.
|
||||
- The tunnel uses HTTP/2 (`--protocol http2`) because TCP/443 to Cloudflare is
|
||||
verified and UDP/7844 is not.
|
||||
@@ -7,7 +7,8 @@
|
||||
#
|
||||
# kubectl -n fch26-onemanarmy create secret generic control-auth \
|
||||
# --from-literal=CONTROL_USER='<control user>' \
|
||||
# --from-literal=CONTROL_PASS='<strong password>'
|
||||
# --from-literal=CONTROL_PASS='<strong password>' \
|
||||
# --from-literal=CONTROL_TOKEN='<random token shared by API and ML>'
|
||||
#
|
||||
# kubectl -n fch26-onemanarmy create secret docker-registry gitea-regcred \
|
||||
# --docker-server=git.noelsabia.dev \
|
||||
@@ -31,6 +32,7 @@ type: Opaque
|
||||
stringData:
|
||||
CONTROL_USER: "REPLACE_ME"
|
||||
CONTROL_PASS: "REPLACE_ME"
|
||||
CONTROL_TOKEN: "REPLACE_ME"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
|
||||
@@ -27,6 +27,7 @@ data:
|
||||
HOST: "0.0.0.0"
|
||||
PORT: "3000"
|
||||
NUXT_TELEMETRY_DISABLED: "1"
|
||||
TMPDIR: "/tmp"
|
||||
NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev"
|
||||
NUXT_API_INTERNAL_BASE: "http://api:8080"
|
||||
NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream"
|
||||
@@ -45,7 +46,6 @@ data:
|
||||
FASTAPI_URL: "http://ml:8000"
|
||||
ML_TIMEOUT: "1.5s"
|
||||
SSE_HEARTBEAT: "15s"
|
||||
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
@@ -56,8 +56,15 @@ metadata:
|
||||
app.kubernetes.io/name: ml
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
data:
|
||||
# The image CMD binds 0.0.0.0:8000; PORT documents the contract.
|
||||
PORT: "8000"
|
||||
DATA_DIR: "/app/data"
|
||||
MODEL_PATH: "/app/model/model.joblib"
|
||||
# These are the names pulsecast.config actually reads. The model and JSON
|
||||
# artifacts are baked into the image; only models/uploads are writable.
|
||||
PULSECAST_ROOT: "/app"
|
||||
PULSECAST_ARTIFACTS: "/app/artifacts"
|
||||
PULSECAST_WEATHER: "/app/artifacts"
|
||||
PULSECAST_MODELS: "/tmp/pulsecast/models"
|
||||
PULSECAST_UPLOADS: "/tmp/pulsecast/uploads"
|
||||
TMPDIR: "/tmp"
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
PYTHONUNBUFFERED: "1"
|
||||
+13
-1
@@ -28,10 +28,16 @@ spec:
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
# Numeric UID is required: a non-numeric image user breaks runAsNonRoot
|
||||
# (verified on SKE with curlimages/curl, 2026-11-14).
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
containers:
|
||||
- name: ml
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1
|
||||
imagePullPolicy: IfNotPresent
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
@@ -46,6 +52,12 @@ spec:
|
||||
value: /tmp
|
||||
- name: MPLCONFIGDIR
|
||||
value: /tmp
|
||||
# Required by FastAPI /train; returns 503 when it is unset.
|
||||
- name: CONTROL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: control-auth
|
||||
key: CONTROL_TOKEN
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
|
||||
+13
-1
@@ -29,10 +29,16 @@ spec:
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
# distroless static:nonroot is UID/GID 65532. Keep it numeric so
|
||||
# runAsNonRoot never depends on the image's user name.
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
containers:
|
||||
- name: api
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1
|
||||
imagePullPolicy: IfNotPresent
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
@@ -56,6 +62,12 @@ spec:
|
||||
secretKeyRef:
|
||||
name: control-auth
|
||||
key: CONTROL_PASS
|
||||
# Forwarded as a Bearer token to FastAPI's internal /train routes.
|
||||
- name: CONTROL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: control-auth
|
||||
key: CONTROL_TOKEN
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
|
||||
@@ -31,11 +31,17 @@ spec:
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
# Numeric UID: the web image must run as (or at least be readable by)
|
||||
# 10001:10001. Do not rely on a named image user.
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
containers:
|
||||
- name: web
|
||||
# Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`.
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1
|
||||
imagePullPolicy: IfNotPresent
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
|
||||
@@ -31,14 +31,22 @@ spec:
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
# cloudflared's image user is 65532:65532; set it numerically.
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
containers:
|
||||
- name: cloudflared
|
||||
# Public Docker Hub image; no Gitea pull secret needed for this pod.
|
||||
image: cloudflare/cloudflared:latest
|
||||
# Pinned rather than :latest so a surprise upstream change cannot break Gate 1.
|
||||
image: cloudflare/cloudflared:2026.11.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- tunnel
|
||||
- --no-autoupdate
|
||||
# SKE smoke confirmed TCP/443 to Cloudflare but did not test
|
||||
# UDP/7844 (QUIC). HTTP/2 avoids depending on untested UDP.
|
||||
- --protocol
|
||||
- http2
|
||||
- --metrics
|
||||
- 0.0.0.0:2000
|
||||
- run
|
||||
|
||||
+594
-176
@@ -2,150 +2,458 @@
|
||||
|
||||
Target: three services (`web`, `api`, `ml`) plus one `cloudflared` Deployment in
|
||||
namespace `fch26-onemanarmy`, reachable over public HTTPS, with Gitea image
|
||||
pulls working.
|
||||
pulls proven before the real images are deployed.
|
||||
|
||||
**The human runs every cluster-mutating command in this document.** The infra
|
||||
agent only writes and validates manifests. No Ingress, cert-manager,
|
||||
external-dns, Jobs, CronJobs, CRDs, Roles, RoleBindings, NetworkPolicies, or
|
||||
other cluster-scoped resources are used.
|
||||
|
||||
## What Gate 1 means
|
||||
|
||||
Gate 1 passes only when **all** of these are true:
|
||||
|
||||
1. The cluster pulls an image from Gitea in `fch26-onemanarmy` using
|
||||
`gitea-regcred` (proven with a hello-world before the real images).
|
||||
2. A Cloudflare Tunnel serves a public HTTPS URL from this namespace.
|
||||
3. `web`, `api`, `ml`, and `cloudflared` are Ready.
|
||||
4. `make smoke SMOKE_SSE=1` passes and a phone on mobile data can open
|
||||
`https://pulse.noelsabia.dev`.
|
||||
5. `https://api.pulse.noelsabia.dev/api/healthz` returns HTTP 200.
|
||||
|
||||
Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest
|
||||
dry-run, or a container that only runs on your laptop.
|
||||
|
||||
## Current external status (2026-11-14 03:50 UTC smoke)
|
||||
|
||||
Findings from `CLUSTER-SMOKE.md`; re-check before trusting this snapshot:
|
||||
|
||||
- **Verified:** cluster access, scheduling, Service DNS/HTTP, probes, Secrets,
|
||||
non-root + read-only-root + `emptyDir`, and egress to Cloudflare,
|
||||
Open-Meteo, and `git.noelsabia.dev`.
|
||||
- **No public DNS yet:** `pulse.noelsabia.dev` and
|
||||
`api.pulse.noelsabia.dev` have no records. That is expected until the
|
||||
Cloudflare Tunnel public hostnames are created.
|
||||
- **No tunnel yet:** Cloudflare Tunnel has not been created.
|
||||
- **Gitea pull auth is untested:** the network path works, but no
|
||||
`gitea-regcred` image pull has succeeded because the read-only token does
|
||||
not exist yet. The hello pod in Step 2 is the first proof.
|
||||
- **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the
|
||||
`cloudflared` manifest uses `--protocol http2` instead of relying on
|
||||
unverified UDP.
|
||||
- **Numeric UIDs are required:** a non-numeric image user breaks
|
||||
`runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in
|
||||
`deploy/k8s/` sets an explicit numeric `runAsUser`/`runAsGroup`.
|
||||
- **Repo status (2026-11-14 ~05:10 UTC):** all three images now build and pass
|
||||
local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp`
|
||||
tmpfs). There is no remaining Dockerfile blocker; the human Gitea/Tunnel
|
||||
steps are what remain.
|
||||
|
||||
## What is possible now vs what is blocked
|
||||
|
||||
| Step | Who | Status |
|
||||
|---|---|---|
|
||||
| Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) |
|
||||
| Gitea push login + hello-world push | human | **ready now**; registry auth still untested |
|
||||
| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **ready now** (needs token) |
|
||||
| Cloudflare Tunnel + public hostnames/DNS | human | **blocked on human** — not created; DNS absent |
|
||||
| `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally |
|
||||
| `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally |
|
||||
| `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally |
|
||||
| Full deploy + smoke + Gate 1 sign-off | human | **ready once the Gitea/Tunnel steps are done** |
|
||||
|
||||
`make build` and `make push` cover all three services; no Dockerfile is
|
||||
missing now. Do not deploy placeholder images under the real service names.
|
||||
|
||||
## Before you start
|
||||
|
||||
Run every block in this document in **one shell**, from this worktree. The
|
||||
commands are bash/zsh compatible.
|
||||
|
||||
```bash
|
||||
cd /home/noel/Documents/Hackathon_2026/infra-k8.git/feat-infra-bootstrap
|
||||
|
||||
export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarmy-sa-kubeconfig.yaml"
|
||||
export NS="fch26-onemanarmy"
|
||||
export CTX="fch26-onemanarmy-sa-context"
|
||||
export REGISTRY="git.noelsabia.dev"
|
||||
export OWNER="noelsabia"
|
||||
export TAG="gate1"
|
||||
|
||||
chmod 600 "$KUBECONFIG"
|
||||
```
|
||||
|
||||
The kubeconfig is **not in this repo**. If it moved, change the path; never
|
||||
commit it or paste its contents anywhere.
|
||||
|
||||
Check the local tools:
|
||||
|
||||
```bash
|
||||
for tool in kubectl docker curl jq make; do
|
||||
command -v "$tool" >/dev/null || echo "MISSING: $tool"
|
||||
done
|
||||
```
|
||||
|
||||
Expected: no `MISSING` lines. `kubeconform`, `k6`, and `yamllint` are optional.
|
||||
|
||||
Check the context and the namespace permissions (read-only):
|
||||
|
||||
```bash
|
||||
kubectl config current-context
|
||||
kubectl --context "$CTX" -n "$NS" auth can-i create deployments
|
||||
kubectl --context "$CTX" -n "$NS" auth can-i create services
|
||||
kubectl --context "$CTX" -n "$NS" auth can-i create ingresses
|
||||
kubectl --context "$CTX" -n "$NS" auth can-i create jobs
|
||||
kubectl --context "$CTX" -n "$NS" get deployments
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
```text
|
||||
fch26-onemanarmy-sa-context
|
||||
yes
|
||||
yes
|
||||
no
|
||||
no
|
||||
No resources found in fch26-onemanarmy namespace.
|
||||
```
|
||||
|
||||
If `deployments` or `services` is not `yes`, or `ingresses`/`jobs` is not
|
||||
`no`, stop and report. Do not work around permissions with cluster-scoped
|
||||
resources.
|
||||
|
||||
## Gate 1 checklist
|
||||
|
||||
Run top to bottom. Stop at the first failure and use
|
||||
[Troubleshooting](#troubleshooting).
|
||||
|
||||
- [ ] 1. Kubeconfig exported, `600`, correct context, namespace reachable.
|
||||
- [ ] 2. Gitea **read-only package token** created and `docker login` succeeds.
|
||||
- [ ] 3. Hello-world image pushed and pulled by the cluster with `gitea-regcred`.
|
||||
- [ ] 4. Secrets created: `gitea-regcred`, `cloudflared`, `control-auth`.
|
||||
- [ ] 5. Real `web`/`api`/`ml` images built, tagged `gate1`, pushed to Gitea.
|
||||
- [ ] 6. `make validate` passes locally.
|
||||
- [ ] 7. Cloudflare Tunnel exists; both public hostnames configured.
|
||||
- [ ] 8. `make deploy CONFIRM=yes` completes and all four Deployments are ready.
|
||||
- [ ] 9. `make smoke SMOKE_SSE=1` passes.
|
||||
- [ ] 10. `https://pulse.noelsabia.dev` opens on a phone over mobile data.
|
||||
- [ ] 11. `https://api.pulse.noelsabia.dev/api/healthz` returns 200.
|
||||
- [ ] 12. Record the deployed image tags and the time in the status note.
|
||||
- [ ] 0. `make gate1-check` shows 0 hard failures.
|
||||
- [ ] 1. Gitea push login succeeds (`Login Succeeded`).
|
||||
- [ ] 2. Read-only package token created; hello image pushed to Gitea.
|
||||
- [ ] 3. `gitea-regcred` created; hello pod is `1/1 Running` from Gitea
|
||||
(the Secret existing is not proof).
|
||||
- [ ] 4. Cloudflare Tunnel created; both public hostnames configured and DNS
|
||||
records resolve.
|
||||
- [ ] 5. `cloudflared` Secret created; logs show `Registered tunnel connection`.
|
||||
- [ ] 6. Public hello route returns HTTP 200 (proves tunnel + DNS + TLS).
|
||||
- [ ] 7. `control-auth` Secret created.
|
||||
- [ ] 8. All three images built, tagged `gate1`, and pushed.
|
||||
- [ ] 9. `make validate` and `make gate1-check` pass.
|
||||
- [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready.
|
||||
- [ ] 11. `make smoke SMOKE_SSE=1` passes.
|
||||
- [ ] 12. Phone over mobile data opens `https://pulse.noelsabia.dev`;
|
||||
`https://api.pulse.noelsabia.dev/api/healthz` returns 200.
|
||||
- [ ] 13. Image tags/digests and the time are recorded.
|
||||
|
||||
## 0. Prerequisites
|
||||
|
||||
The kubeconfig is **not in this repo**. It lives outside git:
|
||||
## Step 0 — read-only readiness and dry-run
|
||||
|
||||
```bash
|
||||
export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarmy-sa-kubeconfig.yaml"
|
||||
export NS=fch26-onemanarmy
|
||||
export CTX=fch26-onemanarmy-sa-context
|
||||
export REGISTRY=git.noelsabia.dev
|
||||
export OWNER=noelsabia
|
||||
export TAG=gate1
|
||||
|
||||
chmod 600 "$KUBECONFIG"
|
||||
make validate
|
||||
make gate1-check
|
||||
```
|
||||
|
||||
Verify read-only first:
|
||||
Expected now (before any human work):
|
||||
|
||||
- `make validate` prints `== kubectl client dry-run (read-only)` and 11
|
||||
`created (dry run)` lines.
|
||||
- `make gate1-check` reports **0 hard failures**, the expected permission
|
||||
matrix, and lists missing Secrets/Deployments as `[pending]`.
|
||||
- Exit code 2 from `make gate1-check` is correct at this stage: it means only
|
||||
human prerequisites remain, not that the manifests are broken.
|
||||
|
||||
Exit codes: `0` ready, `1` hard failure (permissions or manifests), `2` only
|
||||
pending human prerequisites. Use `make gate1-check ALLOW_PENDING=1` when you
|
||||
want exit 0 while prerequisites are still being prepared.
|
||||
|
||||
Neither target mutates the cluster. `make validate` uses
|
||||
`kubectl apply --dry-run=client`; `make gate1-check` only reads
|
||||
configuration/RBAC/secrets and runs the same dry-run.
|
||||
|
||||
## Step 1 — Gitea credentials
|
||||
|
||||
You need **two different credentials**. Do not reuse the read-only token for
|
||||
pushing.
|
||||
|
||||
1. **Push credential:** your normal Gitea account password, or a token with
|
||||
`write:package`. Use a token if the account has 2FA.
|
||||
2. **Cluster pull credential:** a separate token with **only**
|
||||
`read:package`. This becomes `gitea-regcred`; it never needs write access.
|
||||
|
||||
Create the read-only token in Gitea: **Settings → Applications → Generate New
|
||||
Token** (or the package-token screen). Name it `pulsecast-k8s-pull`; scope
|
||||
`read:package`. Copy it once.
|
||||
|
||||
Log in for pushing:
|
||||
|
||||
```bash
|
||||
kubectl config current-context # fch26-onemanarmy-sa-context
|
||||
kubectl -n "$NS" auth can-i create deployments # yes
|
||||
kubectl -n "$NS" auth can-i create services # yes
|
||||
kubectl -n "$NS" auth can-i create ingresses # no
|
||||
kubectl -n "$NS" auth can-i create jobs # no
|
||||
kubectl -n "$NS" get pods,svc,secrets
|
||||
read -rsp 'Gitea push password or write:package token: ' GITEA_PUSH_SECRET; echo
|
||||
printf '%s' "$GITEA_PUSH_SECRET" | docker login "$REGISTRY" -u "$OWNER" --password-stdin
|
||||
```
|
||||
|
||||
If any of the first three answers differ, stop and report; do not work around
|
||||
permissions with cluster-scoped resources.
|
||||
Expected:
|
||||
|
||||
Local tools: `git`, `make`, `bash`, `curl`, `docker`, `kubectl`. Optional:
|
||||
`kubeconform` (schema validation), `jq`, `k6` built with the SSE extension.
|
||||
```text
|
||||
Login Succeeded
|
||||
```
|
||||
|
||||
## 1. Gitea read-only package token
|
||||
|
||||
In Gitea: **Settings → Applications → Generate New Token** (or the package
|
||||
token screen). Scope: read-only package access. Do not reuse the admin token.
|
||||
Store the read-only token in a shell variable for the Secret step:
|
||||
|
||||
```bash
|
||||
read -rsp 'Gitea package token: ' GITEA_TOKEN; echo
|
||||
printf '%s' "$GITEA_TOKEN" | docker login "$REGISTRY" -u "$OWNER" --password-stdin
|
||||
read -rsp 'Gitea read-only package token: ' GITEA_RO_TOKEN; echo
|
||||
```
|
||||
|
||||
A successful login prints `Login Succeeded`. Never put the token in a file,
|
||||
a command that lands in shell history, or this repo.
|
||||
Expected: no output (the value is not echoed).
|
||||
|
||||
## 2. Hello-world image pull (prove the registry path)
|
||||
If login fails with `401 Unauthorized`, the password/token or username is
|
||||
wrong, or 2FA requires a token. Do not put either credential in a file, a
|
||||
command that lands in shell history, or this repo.
|
||||
|
||||
## Step 2 — hello-world image push and pull
|
||||
|
||||
This proves the registry path before spending time on application images.
|
||||
|
||||
```bash
|
||||
docker pull nginxinc/nginx-unprivileged:alpine
|
||||
docker tag nginxinc/nginx-unprivileged:alpine "$REGISTRY/$OWNER/hello:gate1"
|
||||
docker push "$REGISTRY/$OWNER/hello:gate1"
|
||||
|
||||
kubectl -n "$NS" create secret docker-registry gitea-regcred \
|
||||
--docker-server="$REGISTRY" \
|
||||
--docker-username="$OWNER" \
|
||||
--docker-password="$GITEA_TOKEN"
|
||||
|
||||
kubectl -n "$NS" create deployment hello \
|
||||
--image="$REGISTRY/$OWNER/hello:gate1"
|
||||
kubectl -n "$NS" patch deployment hello --type=merge -p \
|
||||
'{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"gitea-regcred"}]}}}}'
|
||||
kubectl -n "$NS" rollout status deployment/hello --timeout=180s
|
||||
kubectl -n "$NS" get pods -l app=hello
|
||||
```
|
||||
|
||||
Pass: the pod is `Running` and `kubectl describe pod` shows the image pulled.
|
||||
Clean up before the real deploy:
|
||||
Expected: the push ends with a digest line, for example:
|
||||
|
||||
```text
|
||||
gate1: digest: sha256:... size: ...
|
||||
```
|
||||
|
||||
If you see `denied: requested access to the resource is denied`, the push
|
||||
credential lacks `write:package` or the owner/package path is wrong. Fix that
|
||||
before continuing.
|
||||
|
||||
Create the pull secret with the **read-only** token:
|
||||
|
||||
```bash
|
||||
kubectl -n "$NS" delete deployment hello
|
||||
kubectl --context "$CTX" -n "$NS" create secret docker-registry gitea-regcred \
|
||||
--docker-server="$REGISTRY" \
|
||||
--docker-username="$OWNER" \
|
||||
--docker-password="$GITEA_RO_TOKEN"
|
||||
```
|
||||
|
||||
If this fails, fix the token/secret before building the application images.
|
||||
Expected: `secret/gitea-regcred created`. If it already exists, delete and
|
||||
recreate it after reading the troubleshooting note:
|
||||
`kubectl --context "$CTX" -n "$NS" delete secret gitea-regcred`.
|
||||
|
||||
## 3. Create the remaining Secrets
|
||||
Deploy the hello image and make the cluster pull it:
|
||||
|
||||
Generate values locally; never commit them.
|
||||
```bash
|
||||
kubectl --context "$CTX" -n "$NS" create deployment hello \
|
||||
--image="$REGISTRY/$OWNER/hello:gate1"
|
||||
|
||||
kubectl --context "$CTX" -n "$NS" patch deployment hello --type=merge -p \
|
||||
'{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"gitea-regcred"}]}}}}'
|
||||
|
||||
kubectl --context "$CTX" -n "$NS" rollout status deployment/hello --timeout=180s
|
||||
kubectl --context "$CTX" -n "$NS" get pods -l app=hello -o wide
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
```text
|
||||
deployment "hello" successfully rolled out
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
hello-xxxxxxxxxx-xxxxx 1/1 Running 0 20s
|
||||
```
|
||||
|
||||
**This is the first real pass/fail.** A `gitea-regcred` Secret existing is not
|
||||
proof: the Gate 1 Gitea item is complete only when this hello pod is `1/1
|
||||
Running` after pulling from Gitea. If the pod is not `Running`, stop.
|
||||
`ImagePullBackOff` means the token, secret namespace, image tag, or registry
|
||||
DNS is wrong. Diagnose with:
|
||||
|
||||
```bash
|
||||
kubectl --context "$CTX" -n "$NS" describe pod -l app=hello
|
||||
```
|
||||
|
||||
Keep the hello Deployment until Step 6 proves the public route. Create a
|
||||
Service for it so Cloudflare has something to reach:
|
||||
|
||||
```bash
|
||||
kubectl --context "$CTX" -n "$NS" expose deployment hello \
|
||||
--port=8080 --target-port=8080
|
||||
```
|
||||
|
||||
Expected: `service/hello exposed`.
|
||||
|
||||
## Step 3 — Cloudflare Tunnel
|
||||
|
||||
As of the 03:50 UTC smoke, neither public hostname has a DNS record. That is
|
||||
expected before the tunnel route exists; do not add manual A/AAAA records.
|
||||
|
||||
```bash
|
||||
getent hosts pulse.noelsabia.dev || echo 'pulse: no DNS record yet (expected)'
|
||||
getent hosts api.pulse.noelsabia.dev || echo 'api: no DNS record yet (expected)'
|
||||
```
|
||||
|
||||
1. Open Cloudflare **Zero Trust → Networks → Tunnels → Create a tunnel**.
|
||||
2. Connector: **Cloudflared**. Name: `pulsecast-gate1`.
|
||||
3. Save and copy the tunnel token. It is shown once.
|
||||
4. Open the tunnel's **Public Hostnames** tab and add:
|
||||
|
||||
| Public hostname | Service |
|
||||
|---|---|
|
||||
| `pulse.noelsabia.dev` | `http://hello:8080` (temporary, Gate 1-A) |
|
||||
| `api.pulse.noelsabia.dev` | `http://api:8080` (works after Step 10) |
|
||||
|
||||
For the final Gate 1-B route, change `pulse.noelsabia.dev` to
|
||||
`http://web:3000` after the web image is deployed. If you already have the
|
||||
web image, you may set `web:3000` now and skip the hello route; then the
|
||||
public hello check in Step 6 is not available, but the tunnel proof still
|
||||
happens when the real web pod is Ready.
|
||||
|
||||
5. The hostnames must be in the Cloudflare zone for `noelsabia.dev`.
|
||||
Cloudflare creates the DNS records for a tunnel hostname; do **not** add
|
||||
manual A/AAAA records. If the dashboard refuses to save a hostname because
|
||||
a conflicting A/AAAA/CNAME already exists for that exact name, delete the
|
||||
conflicting record first.
|
||||
6. Do not add an Ingress. The tunnel is the only public path.
|
||||
|
||||
After saving both hostnames, DNS should resolve to Cloudflare:
|
||||
|
||||
```bash
|
||||
getent hosts pulse.noelsabia.dev
|
||||
getent hosts api.pulse.noelsabia.dev
|
||||
```
|
||||
|
||||
Expected: one or more Cloudflare IPv4/IPv6 addresses. If there is still no
|
||||
record, the public hostname was not saved, it is in the wrong zone, or a
|
||||
conflicting record exists. Fix that in the dashboard before testing HTTPS;
|
||||
`curl` cannot succeed without DNS.
|
||||
|
||||
Create the tunnel Secret (the token is not echoed):
|
||||
|
||||
```bash
|
||||
read -rsp 'Cloudflare tunnel token: ' TUNNEL_TOKEN; echo
|
||||
kubectl --context "$CTX" -n "$NS" create secret generic cloudflared \
|
||||
--from-literal=TUNNEL_TOKEN="$TUNNEL_TOKEN"
|
||||
```
|
||||
|
||||
Expected: `secret/cloudflared created`.
|
||||
|
||||
Deploy only the tunnel first. This does not need the application images:
|
||||
|
||||
```bash
|
||||
kubectl --context "$CTX" -n "$NS" apply -f deploy/k8s/40-cloudflared.yaml
|
||||
kubectl --context "$CTX" -n "$NS" rollout status deployment/cloudflared --timeout=180s
|
||||
kubectl --context "$CTX" -n "$NS" logs deployment/cloudflared --tail=100 \
|
||||
| grep 'Registered tunnel connection' || echo 'NO TUNNEL CONNECTION'
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
```text
|
||||
deployment.apps/cloudflared created
|
||||
deployment "cloudflared" successfully rolled out
|
||||
... Registered tunnel connection ...
|
||||
```
|
||||
|
||||
(Usually four `Registered tunnel connection` lines, one per edge.)
|
||||
|
||||
The manifest deliberately uses `--protocol http2`: TCP/443 to Cloudflare is
|
||||
verified, UDP/7844 (QUIC) is not. Do not switch to QUIC during Gate 1. If you
|
||||
do test QUIC later, revert to HTTP/2 on any connection error.
|
||||
|
||||
If `cloudflared` is in `CrashLoopBackOff`, the token is invalid/expired or the
|
||||
tunnel was deleted:
|
||||
|
||||
```bash
|
||||
kubectl --context "$CTX" -n "$NS" logs deployment/cloudflared --previous
|
||||
kubectl --context "$CTX" -n "$NS" describe pod -l app.kubernetes.io/name=cloudflared
|
||||
```
|
||||
|
||||
## Step 4 — control-auth Secret
|
||||
|
||||
The API reads `CONTROL_USER` and `CONTROL_PASS` for browser control auth.
|
||||
Both API and ML also need the same `CONTROL_TOKEN`: the API forwards it as a
|
||||
Bearer token to FastAPI's internal `/train` routes. Generate both locally;
|
||||
never commit them.
|
||||
|
||||
```bash
|
||||
read -rsp 'Control password: ' CONTROL_PASS; echo
|
||||
kubectl -n "$NS" create secret generic control-auth \
|
||||
CONTROL_TOKEN="$(python3 -c 'import secrets; print(secrets.token_hex(24))')"
|
||||
kubectl --context "$CTX" -n "$NS" create secret generic control-auth \
|
||||
--from-literal=CONTROL_USER=control \
|
||||
--from-literal=CONTROL_PASS="$CONTROL_PASS"
|
||||
|
||||
# Token from Cloudflare Zero Trust → Networks → Tunnels → your tunnel.
|
||||
read -rsp 'Cloudflare tunnel token: ' TUNNEL_TOKEN; echo
|
||||
kubectl -n "$NS" create secret generic cloudflared \
|
||||
--from-literal=TUNNEL_TOKEN="$TUNNEL_TOKEN"
|
||||
|
||||
kubectl -n "$NS" get secrets
|
||||
--from-literal=CONTROL_PASS="$CONTROL_PASS" \
|
||||
--from-literal=CONTROL_TOKEN="$CONTROL_TOKEN"
|
||||
```
|
||||
|
||||
`make secret-commands` prints the same commands with placeholders.
|
||||
Expected: `secret/control-auth created`.
|
||||
|
||||
## 4. Build and push the three images
|
||||
The token is shared by API and ML, not the browser password. Without it
|
||||
FastAPI `/train` returns 503 and the API's `/api/train` proxy cannot work.
|
||||
|
||||
The manifests expect:
|
||||
`make secret-commands` prints all three Secret commands with placeholders; it
|
||||
creates nothing.
|
||||
|
||||
| Service | Image | Port | Health paths |
|
||||
|---|---|---|---|
|
||||
| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `/` |
|
||||
| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `/api/healthz`, `/api/readyz` |
|
||||
| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `/healthz`, `/readyz` |
|
||||
## Step 5 — build and push the three images (Phase B)
|
||||
|
||||
Check whether the Dockerfiles exist:
|
||||
|
||||
```bash
|
||||
make build # builds all three from the service worktrees
|
||||
make push # docker push to Gitea
|
||||
test -f ../../web-nuxt.git/feat-web-bootstrap/Dockerfile && echo web-ok || echo web-blocked
|
||||
test -f ../../api-go.git/feat-api-bootstrap/Dockerfile && echo api-ok || echo api-blocked
|
||||
test -f ../../ML-py.git/feat-ml-bootstrap/Dockerfile && echo ml-ok || echo ml-blocked
|
||||
```
|
||||
|
||||
`make build` auto-detects the service worktrees under
|
||||
`/home/noel/Documents/Hackathon_2026/worktrees/`. Override when needed:
|
||||
Expected **now** (all three Dockerfiles landed on 2026-11-14):
|
||||
|
||||
```text
|
||||
web-ok
|
||||
api-ok
|
||||
ml-ok
|
||||
```
|
||||
|
||||
Build all three:
|
||||
|
||||
```bash
|
||||
make build
|
||||
```
|
||||
|
||||
All three images were verified locally with the manifest's runtime
|
||||
constraints:
|
||||
|
||||
- API: distroless `static:nonroot`, `--user 65532:65532`, read-only root,
|
||||
`/tmp` tmpfs; `/api/healthz` and `/api/readyz` returned 200.
|
||||
- ML: image `USER app` is non-numeric, but the manifest overrides it with
|
||||
`runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/healthz` and
|
||||
`/readyz` returned 200 and `/meta` loaded the baked model.
|
||||
- Web: image `USER node` is non-numeric, but the manifest overrides it with
|
||||
`runAsUser: 10001`; with read-only root and `/tmp` tmpfs, `/` returned
|
||||
HTTP 200 and the runtime `NUXT_PUBLIC_API_BASE` appeared in the HTML.
|
||||
- No image was pushed or deployed by the infra agent.
|
||||
|
||||
Push:
|
||||
|
||||
```bash
|
||||
make push
|
||||
```
|
||||
|
||||
Expected: each `docker push` ends with a digest. If authentication expired,
|
||||
repeat the push login from Step 1.
|
||||
|
||||
The Makefile auto-detects the service worktrees:
|
||||
|
||||
```text
|
||||
../../web-nuxt.git/feat-web-bootstrap
|
||||
../../api-go.git/feat-api-bootstrap
|
||||
../../ML-py.git/feat-ml-bootstrap
|
||||
```
|
||||
|
||||
Override when the layout differs:
|
||||
|
||||
```bash
|
||||
make build WEB_DIR=/path/to/web-nuxt API_DIR=/path/to/api-go ML_DIR=/path/to/ML-py
|
||||
```
|
||||
|
||||
If `OWNER` or `TAG` differs from the manifest defaults, either update
|
||||
`deploy/k8s/*.yaml` before `make deploy`, or deploy first and then run the
|
||||
guarded image update:
|
||||
If `OWNER` or `TAG` differs from the manifests, update `deploy/k8s/*.yaml`
|
||||
before `make deploy`, or deploy first and then run the guarded image update:
|
||||
|
||||
```bash
|
||||
make set-image CONFIRM=yes OWNER=<owner> TAG=<tag>
|
||||
@@ -155,171 +463,281 @@ make set-image CONFIRM=yes OWNER=<owner> TAG=<tag>
|
||||
|
||||
Each service image must:
|
||||
|
||||
- listen on `0.0.0.0` at the port above;
|
||||
- run as a non-root user (numeric UID, no shell required);
|
||||
- listen on `0.0.0.0` at the port below;
|
||||
- run as a **non-root numeric UID** (no named user, no shell required);
|
||||
- write only to `/tmp` (the pod root filesystem is read-only);
|
||||
- start with the environment in `deploy/k8s/00-config.yaml`;
|
||||
- be self-contained: no runtime download of data, model, fonts, or packages.
|
||||
|
||||
| Service | Image | Port | UID:GID | Health paths |
|
||||
|---|---|---|---|---|
|
||||
| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` |
|
||||
| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` |
|
||||
| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` |
|
||||
|
||||
The manifests set `runAsUser`/`runAsGroup` explicitly, so the image's named
|
||||
user is irrelevant; the files must be readable by that numeric UID. `api`
|
||||
uses distroless `static:nonroot` (65532); web and ml use 10001. If a Dockerfile
|
||||
genuinely needs another UID, change the manifest and this table together, and
|
||||
keep the value numeric.
|
||||
|
||||
`api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`,
|
||||
`CONTROL_USER`, `CONTROL_PASS`, `SSE_HEARTBEAT`, and `ML_TIMEOUT`.
|
||||
`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in or
|
||||
provided at runtime; `ml` must find its model and artifacts under the paths its
|
||||
Dockerfile defines. The env names in the manifests are the integration point:
|
||||
change them here first if a service needs a different name.
|
||||
`CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and
|
||||
`ML_TIMEOUT`.
|
||||
`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in
|
||||
or read it at runtime from the ConfigMap; the browser must use the public API
|
||||
hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`,
|
||||
`PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and
|
||||
`CONTROL_TOKEN`; its image CMD binds `0.0.0.0:8000`. The env names in the
|
||||
manifests are the integration point: change them here first if a service needs
|
||||
a different name. The ML image currently uses a named `app` user; the manifest
|
||||
overrides it with numeric UID 10001, which was verified locally.
|
||||
|
||||
## 5. Cloudflare Tunnel
|
||||
## Step 6 — prove public HTTPS with hello
|
||||
|
||||
1. Zero Trust → **Networks → Tunnels → Create a tunnel**.
|
||||
2. Connector: **Cloudflared**. Name: `pulsecast-gate1`.
|
||||
3. Copy the tunnel token (it is shown once). Store it in the `cloudflared`
|
||||
Secret (step 3).
|
||||
4. Add public hostnames:
|
||||
Do this **before** deleting hello. It proves tunnel + DNS + TLS without
|
||||
depending on web/ml.
|
||||
|
||||
| Public hostname | Service |
|
||||
|---|---|
|
||||
| `pulse.noelsabia.dev` | `http://web:3000` |
|
||||
| `api.pulse.noelsabia.dev` | `http://api:8080` |
|
||||
```bash
|
||||
curl -fsS -o /dev/null -w 'hello %{http_code}\n' https://pulse.noelsabia.dev/
|
||||
```
|
||||
|
||||
The hostnames must be in the Cloudflare zone for `noelsabia.dev`. The
|
||||
service URL uses the Kubernetes Service name and port inside the namespace;
|
||||
no Ingress is involved.
|
||||
5. For the first smoke test before the app images exist, temporarily point
|
||||
`pulse.noelsabia.dev` at `http://hello:8080`, then switch it to
|
||||
`http://web:3000` after step 7.
|
||||
Expected: `hello 200`.
|
||||
|
||||
DNS records for the public hostnames are created by Cloudflare; do not add
|
||||
manual A/AAAA records for a tunnel hostname.
|
||||
If you get `502`, `530`, or `1033`, the edge cannot reach the Service. Check:
|
||||
|
||||
## 6. Validate before applying
|
||||
```bash
|
||||
kubectl --context "$CTX" -n "$NS" get endpoints hello
|
||||
kubectl --context "$CTX" -n "$NS" get svc hello
|
||||
kubectl --context "$CTX" -n "$NS" logs deployment/cloudflared --tail=50
|
||||
```
|
||||
|
||||
The hostname route must be `http://hello:8080` and the hello Service must have
|
||||
an endpoint. Fix the dashboard route or the Service; do not add an Ingress.
|
||||
|
||||
After the public hello check passes, delete it so it cannot mask a web problem:
|
||||
|
||||
```bash
|
||||
kubectl --context "$CTX" -n "$NS" delete service hello
|
||||
kubectl --context "$CTX" -n "$NS" delete deployment hello
|
||||
```
|
||||
|
||||
Expected: `service "hello" deleted` and `deployment.apps "hello" deleted`.
|
||||
|
||||
`https://pulse.noelsabia.dev` will now return an edge error until the real web
|
||||
Deployment is Ready. That is expected; do not "fix" it with an Ingress.
|
||||
|
||||
## Step 7 — validate and check readiness again
|
||||
|
||||
```bash
|
||||
make validate
|
||||
make gate1-check
|
||||
```
|
||||
|
||||
This is read-only:
|
||||
Expected once Secrets and Deployments exist:
|
||||
|
||||
- `kubeconform -strict` validates the resource schemas if installed;
|
||||
- otherwise, if a cluster is reachable, `kubectl apply --dry-run=client`
|
||||
validates against the API server;
|
||||
- if neither is available it parses the YAML and prints a warning that schema
|
||||
validation was skipped.
|
||||
- `make validate`: 11 resources created (dry run);
|
||||
- `make gate1-check`: 0 hard failures, 0 pending, `RESULT: READY for Gate 1`
|
||||
(exit 0).
|
||||
|
||||
Never run `kubectl apply` / `delete` from an agent session. The commands below
|
||||
are for the human.
|
||||
Before the deploy, pending Deployments keep it at exit 2. That is fine.
|
||||
|
||||
## 7. Deploy
|
||||
## Step 8 — deploy
|
||||
|
||||
```bash
|
||||
make deploy-check # read-only: required Secrets exist
|
||||
make deploy CONFIRM=yes # applies deploy/k8s and waits for rollouts
|
||||
make deploy-check
|
||||
make deploy CONFIRM=yes
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
```text
|
||||
required Secrets and keys present
|
||||
... created/configured ...
|
||||
deployment "ml" successfully rolled out
|
||||
deployment "api" successfully rolled out
|
||||
deployment "web" successfully rolled out
|
||||
deployment "cloudflared" successfully rolled out
|
||||
```
|
||||
|
||||
`make deploy` fails closed: without `CONFIRM=yes` it changes nothing. It does
|
||||
not create the namespace or secrets.
|
||||
not create the namespace or Secrets. It applies all 11 manifests, including
|
||||
the ConfigMaps.
|
||||
|
||||
If the three service images are not available yet, deploy only the tunnel and
|
||||
use the hello service to prove public HTTPS; return to this step when the real
|
||||
images exist. Do not deploy a placeholder image under the real service names.
|
||||
If the real images do not exist, the rollouts fail with `ImagePullBackOff`.
|
||||
Stop and finish Step 5; do not deploy a placeholder under `web`, `api`, or
|
||||
`ml`.
|
||||
|
||||
Check the result:
|
||||
Check the state:
|
||||
|
||||
```bash
|
||||
kubectl -n "$NS" get deploy,pods,svc
|
||||
kubectl -n "$NS" get pods -o wide
|
||||
kubectl -n "$NS" logs deployment/cloudflared --tail=50 | grep 'Registered tunnel connection'
|
||||
make status
|
||||
kubectl --context "$CTX" -n "$NS" get pods -o wide
|
||||
kubectl --context "$CTX" -n "$NS" logs deployment/cloudflared --tail=50 \
|
||||
| grep 'Registered tunnel connection' || echo 'NO TUNNEL CONNECTION'
|
||||
```
|
||||
|
||||
Expected: `web`, `api`, `ml`, `cloudflared` each `1/1`; Services `web`, `api`,
|
||||
`ml` are `ClusterIP`; `cloudflared` has no Service (it dials out).
|
||||
Expected: `web`, `api`, `ml`, and `cloudflared` each `1/1`; Services `web`,
|
||||
`api`, and `ml` are `ClusterIP`; `cloudflared` has no Service.
|
||||
|
||||
## 8. Verify
|
||||
### Re-pushing the same tag
|
||||
|
||||
Public checks (also runnable from a phone on mobile data):
|
||||
The manifests use `imagePullPolicy: Always` while the tag stays `gate1`. After
|
||||
rebuilding and pushing the same tag, force a re-pull:
|
||||
|
||||
```bash
|
||||
curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/
|
||||
curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz
|
||||
curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz
|
||||
curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}'
|
||||
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20
|
||||
make restart CONFIRM=yes
|
||||
```
|
||||
|
||||
The SSE output must start with `retry: 3000`, contain a `scenario` event
|
||||
immediately, and show a `ping` within 20 s.
|
||||
This restarts `web`, `api`, and `ml` and waits for the rollout. Do not use it
|
||||
during the demo window unless you are fixing a blocker.
|
||||
|
||||
Full smoke test, including the ML service through a temporary
|
||||
`kubectl port-forward`:
|
||||
## Step 9 — verify public HTTPS and SSE
|
||||
|
||||
Full smoke, including the ML service through a short `kubectl port-forward`:
|
||||
|
||||
```bash
|
||||
make smoke SMOKE_SSE=1
|
||||
```
|
||||
|
||||
Local variant once the three dev servers are running:
|
||||
Expected: each check prints `PASS`, the summary ends with `0 failed`, and TLS
|
||||
verification passes (`ssl_verify_result=0`). If the ML port-forward cannot
|
||||
start, set `ML_URL` or `SKIP_ML=1`; do not silently drop the ML check from the
|
||||
sign-off.
|
||||
|
||||
Manual public checks (also run from a phone on mobile data):
|
||||
|
||||
```bash
|
||||
make smoke-local
|
||||
curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/
|
||||
curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz
|
||||
curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz
|
||||
curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}'
|
||||
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20
|
||||
```
|
||||
|
||||
In-cluster connectivity (optional; creates and removes one short-lived pod):
|
||||
Expected:
|
||||
|
||||
```text
|
||||
web 200
|
||||
api 200
|
||||
ready 200
|
||||
{
|
||||
"data_version": "...",
|
||||
"model_version": "..."
|
||||
}
|
||||
retry: 3000
|
||||
|
||||
event: scenario
|
||||
id: ...
|
||||
data: {...}
|
||||
|
||||
event: ping
|
||||
data: {"t":"..."}
|
||||
```
|
||||
|
||||
The SSE output must start with `retry: 3000`, contain a `scenario` event
|
||||
immediately, and show a `ping` within 20 seconds. If events are delayed
|
||||
through Cloudflare, see the SSE row in Troubleshooting.
|
||||
|
||||
In-cluster connectivity (optional; **creates and removes one short-lived
|
||||
pod**, so the human runs it):
|
||||
|
||||
```bash
|
||||
kubectl -n "$NS" run gate1-netcheck --rm -it --restart=Never \
|
||||
--image=curlimages/curl:latest --command -- sh -c \
|
||||
kubectl --context "$CTX" -n "$NS" run gate1-netcheck --rm -i --restart=Never \
|
||||
--image=curlimages/curl:latest \
|
||||
--overrides='{"spec":{"securityContext":{"runAsNonRoot":true,"runAsUser":1000,"runAsGroup":1000}}}' \
|
||||
--command -- sh -c \
|
||||
'curl -fsS http://api:8080/api/healthz && \
|
||||
curl -fsS http://ml:8000/healthz && \
|
||||
curl -fsS http://web:3000/ >/dev/null && echo connectivity-ok'
|
||||
```
|
||||
|
||||
TLS is verified by `make smoke` (`ssl_verify_result=0`). Certificates are
|
||||
issued by Cloudflare at the edge; there is no cert-manager resource.
|
||||
Expected: `{"ok":true}{"ok":true}connectivity-ok` (order may vary with the
|
||||
shell output). If the namespace's Pod Security Admission blocks this ad-hoc
|
||||
pod, skip it and use `kubectl exec deploy/web -- ...` or the smoke test; do not
|
||||
weaken the application Deployments to make a diagnostic pod run.
|
||||
|
||||
## 9. Record the Gate 1 result
|
||||
Finally, confirm the checker is green end-to-end:
|
||||
|
||||
Write down, in the status note or README:
|
||||
```bash
|
||||
make gate1-check CHECK_PUBLIC=1
|
||||
```
|
||||
|
||||
- deploy time and commit hash;
|
||||
- image tags actually deployed (`kubectl -n "$NS" get deploy -o jsonpath=...`);
|
||||
- output of `make smoke SMOKE_SSE=1`;
|
||||
- public URLs and the device/network used;
|
||||
Expected: `RESULT: READY for Gate 1` (exit 0).
|
||||
|
||||
## Step 10 — record the Gate 1 result
|
||||
|
||||
Record in the status note, README, or commit message:
|
||||
|
||||
```bash
|
||||
date -u +'%Y-%m-%dT%H:%M:%SZ'
|
||||
git rev-parse HEAD
|
||||
kubectl --context "$CTX" -n "$NS" get deploy -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.template.spec.containers[0].image}{"\n"}{end}'
|
||||
```
|
||||
|
||||
Also save:
|
||||
|
||||
- the output of `make smoke SMOKE_SSE=1`;
|
||||
- the public URLs and the device/network used (phone on mobile data);
|
||||
- any warning that remains.
|
||||
|
||||
Image digests are in the `docker push` output; tags alone are not enough if
|
||||
you need to reproduce the exact deploy.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Check | Fix |
|
||||
|---|---|---|
|
||||
| `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod <pod>`; `kubectl -n "$NS" get secret gitea-regcred` | Token is read-only package scoped; image tag exists; secret is in the same namespace. |
|
||||
| `CreateContainerConfigError` | `kubectl -n "$NS" describe pod <pod>` | A referenced Secret key is missing (`control-auth` or `cloudflared`). Recreate the Secret. |
|
||||
| `cloudflared` CrashLoopBackOff | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. |
|
||||
| Public 502 / 1033 | `kubectl -n "$NS" get endpoints web api`; cloudflared logs | Service has no ready endpoints, wrong port, or hostname route points at the wrong service. |
|
||||
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `kubectl -n "$NS" exec deploy/api -- ls /app/data` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. |
|
||||
| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds 120 s or the model path is wrong. Fix the image/path; startupProbe allows 120 s. |
|
||||
| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing build-time API base or server error. Rebuild with `NUXT_PUBLIC_API_BASE`. |
|
||||
| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering; the web client falls back to polling `/api/scenario` every 5 s. Keep the heartbeat at 15 s. |
|
||||
| `make deploy-check` fails | `kubectl -n "$NS" get secrets` | Create the missing Secret; do not commit it. |
|
||||
| `docker login` → `401 Unauthorized` | Username and credential; 2FA | Use a token with `write:package` (not the read-only token), or the account password if 2FA is off. |
|
||||
| `docker push` → `denied: requested access to the resource is denied` | Push credential scope; owner/package path | Use a `write:package` token; make sure `$OWNER` matches the Gitea image owner. |
|
||||
| `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod -l app=hello` (and the real pod); `kubectl -n "$NS" get secret gitea-regcred` | The token must have `read:package`; the Secret must be in `fch26-onemanarmy`; the tag must exist in Gitea. Do not print the secret data. |
|
||||
| `CreateContainerConfigError` | `kubectl -n "$NS" describe pod <pod>` | A referenced Secret key is missing (`control-auth`: `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`) or the `cloudflared` Secret lacks `TUNNEL_TOKEN`. Recreate the Secret. |
|
||||
| `container has runAsNonRoot and image will run as root` or a non-numeric-user error | `kubectl -n "$NS" get pod <pod> -o yaml`; inspect `securityContext` | Keep `runAsNonRoot: true` and set a positive numeric `runAsUser`/`runAsGroup` (SKE caveat). The manifests already do; do not "fix" this by disabling `runAsNonRoot`. |
|
||||
| Public hostname has no DNS record | `getent hosts pulse.noelsabia.dev` / `getent hosts api.pulse.noelsabia.dev` | Save the public hostname in the tunnel dashboard in the `noelsabia.dev` zone; remove a conflicting A/AAAA/CNAME for that exact name. Do not add manual records for a tunnel hostname. |
|
||||
| `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. |
|
||||
| `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. |
|
||||
| Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. |
|
||||
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api.pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. |
|
||||
| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. |
|
||||
| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. |
|
||||
| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. |
|
||||
| `make gate1-check` exits 2 | Output lists `[pending]` Secrets/Deployments | Complete the human steps above. Exit 2 is not a manifest failure. |
|
||||
| `make deploy` refuses | `CONFIRM` is empty | Re-run `make deploy CONFIRM=yes`; this is the safety guard. |
|
||||
| `kubectl` → `Forbidden` | `kubectl config current-context` | Use `--context "$CTX"`. Do not create Roles/Ingress/Jobs as a workaround; escalate instead. |
|
||||
| `make validate` fails | Dry-run output | Fix the manifest; do not apply it to see whether it works. |
|
||||
|
||||
## Rollback
|
||||
|
||||
No cluster-scoped resources are created, so rollback is namespace-local:
|
||||
|
||||
```bash
|
||||
kubectl -n "$NS" rollout undo deployment/web
|
||||
kubectl -n "$NS" rollout undo deployment/api
|
||||
kubectl -n "$NS" rollout undo deployment/ml
|
||||
kubectl -n "$NS" rollout undo deployment/cloudflared
|
||||
kubectl -n "$NS" rollout status deployment/web --timeout=180s
|
||||
kubectl --context "$CTX" -n "$NS" rollout undo deployment/web
|
||||
kubectl --context "$CTX" -n "$NS" rollout undo deployment/api
|
||||
kubectl --context "$CTX" -n "$NS" rollout undo deployment/ml
|
||||
kubectl --context "$CTX" -n "$NS" rollout undo deployment/cloudflared
|
||||
kubectl --context "$CTX" -n "$NS" rollout status deployment/web --timeout=180s
|
||||
```
|
||||
|
||||
If the tunnel token or route is wrong, fix the Secret / Cloudflare dashboard
|
||||
and restart `cloudflared`; do not create an Ingress as a workaround.
|
||||
and restart `cloudflared`; do not create an Ingress as a workaround. Delete any
|
||||
leftover hello resources:
|
||||
|
||||
```bash
|
||||
kubectl --context "$CTX" -n "$NS" delete service hello --ignore-not-found
|
||||
kubectl --context "$CTX" -n "$NS" delete deployment hello --ignore-not-found
|
||||
```
|
||||
|
||||
## Security notes
|
||||
|
||||
- Never commit the kubeconfig, tunnel token, control password, Gitea token, or
|
||||
any `.env` file. `.gitignore` covers the usual filenames; check
|
||||
`git status` before every commit.
|
||||
- The kubeconfig must stay at mode `600` and outside the repos. It was shared
|
||||
in chat: rotate/revoke the service-account token after the event.
|
||||
- Rotate the Gitea package token and the control password after the event.
|
||||
Rotate the service-account token after the event; it was shared in chat.
|
||||
- The kubeconfig must stay at mode `600` and outside the repos.
|
||||
- `make smoke` prints no secrets; do not add `set -x` to the scripts with a
|
||||
live `KUBECONFIG`.
|
||||
- Use the read-only package token only for `gitea-regcred`; use a separate
|
||||
write credential for pushes.
|
||||
- `make smoke` and `make gate1-check` print no secrets; do not add `set -x` to
|
||||
scripts while a live `KUBECONFIG` or token is exported.
|
||||
- Stop the tunnel and rotate the Gitea token if either leaks. Deleting the
|
||||
tunnel in Cloudflare invalidates its token.
|
||||
+30
-11
@@ -7,17 +7,22 @@ and reports connection count plus p95 event latency.
|
||||
|
||||
Native k6 buffers a complete HTTP response, so it cannot parse a stream that
|
||||
never ends. The script imports the community `k6/x/sse` extension
|
||||
(`github.com/phymbert/xk6-sse`). Newer k6 releases resolve community extensions
|
||||
automatically. If the stock `grafana/k6` image rejects the import, build a
|
||||
binary with the extension:
|
||||
(`github.com/phymbert/xk6-sse`), which a stock k6 binary does not include.
|
||||
|
||||
Build the pinned binary once:
|
||||
|
||||
```bash
|
||||
make k6-build # creates k6/k6-sse (git-ignored)
|
||||
K6=./k6/k6-sse make load
|
||||
./k6/k6-sse version # expect k6 v2.0.0 + xk6-sse v0.2.0
|
||||
```
|
||||
|
||||
`make load` uses, in order: `k6` on `PATH`, `k6/k6-sse`, then
|
||||
`docker run --rm -i grafana/k6`.
|
||||
The build pins **k6 v2.0.0**. On k6 v2.4.0 the same `xk6-sse` v0.2.0 build
|
||||
fails at runtime with `TypeError: Object has no member 'on'` (the extension's
|
||||
client methods are not exposed); v2.0.0 was verified locally on 2026-11-14. If
|
||||
you bump `K6_BUILD_VERSION`, rerun the script against a test stream first.
|
||||
|
||||
`make load` uses, in order: `K6` if set, `k6/k6-sse`, then `k6` on `PATH`.
|
||||
A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
|
||||
|
||||
## Procedure
|
||||
|
||||
@@ -35,7 +40,7 @@ K6=./k6/k6-sse make load
|
||||
|
||||
```bash
|
||||
make load
|
||||
# or: API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js
|
||||
# or: API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
|
||||
```
|
||||
|
||||
`make load` writes `k6/results/summary-<timestamp>.json` (git-ignored).
|
||||
@@ -56,25 +61,35 @@ K6=./k6/k6-sse make load
|
||||
|
||||
4. Record these numbers in the paper/README from the saved summary:
|
||||
- `sse_connections_opened` count (must be 120)
|
||||
- `sse_connections_active` max (must reach 120)
|
||||
- `sse_connections_closed` count (must equal connections opened)
|
||||
- `sse_open_success` rate (must be >= 99%)
|
||||
- `sse_event_latency` p95 (target < 2000 ms)
|
||||
- `sse_connection_errors` (target 0)
|
||||
- `sse_events_received` total
|
||||
|
||||
## Local validation before the cluster run
|
||||
|
||||
The script was validated locally on 2026-11-14 with the pinned k6 v2.0.0 build
|
||||
against a mock SSE stream and the real API image (3 VUs, 1 heartbeat each):
|
||||
connections opened and closed cleanly, open success 100%. The real 120-client
|
||||
run still requires the deployed API, because localhost results say nothing
|
||||
about Cloudflare buffering or SKE scheduling.
|
||||
|
||||
## Pass criteria
|
||||
|
||||
| Metric | Target |
|
||||
|---|---|
|
||||
| Connections opened | 120 |
|
||||
| Peak active connections | 120 |
|
||||
| Connections closed (stream ended) | 120 |
|
||||
| Open success rate | >= 99% |
|
||||
| p95 event latency | < 2000 ms |
|
||||
| Unexpected connection errors | 0 |
|
||||
| Duration | 5 minutes |
|
||||
|
||||
The k6 thresholds enforce the first, third, and fourth; the others are read
|
||||
from the summary. A p95 above 2 s means Cloudflare, the Go SSE hub, or the
|
||||
The k6 thresholds enforce connections opened, open success, and p95 latency;
|
||||
the others are read from the summary. A final `sse_connections_closed` below
|
||||
120 means at least one opened VU did not finish; inspect
|
||||
`sse_connection_errors` for unexpected errors. A p95 above 2 s means Cloudflare, the Go SSE hub, or the
|
||||
client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s
|
||||
heartbeat, and whether the connection is HTTP/2-buffered at the edge.
|
||||
|
||||
@@ -83,6 +98,10 @@ heartbeat, and whether the connection is HTTP/2-buffered at the edge.
|
||||
- `sse_event_latency` uses the server timestamp in `data.t` (ping) or `data.at`
|
||||
(scenario). It includes clock skew between the k6 host and the pod; keep both
|
||||
on NTP. Negative or > 1 h values are ignored.
|
||||
- The initial `scenario` event on connect replays the current state and carries
|
||||
the original state-change time, not the send time. The script excludes that
|
||||
first scenario event per VU; only scenario changes broadcast during the run
|
||||
are sampled by `sse_scenario_latency`. Heartbeat pings always count.
|
||||
- Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends
|
||||
cleanly instead of waiting for the 7-minute request timeout.
|
||||
- One API replica means all clients share one SSE hub; a pod restart drops
|
||||
|
||||
@@ -2,17 +2,17 @@
|
||||
//
|
||||
// The native k6 http module buffers the whole response, so it cannot read
|
||||
// Server-Sent Events incrementally. This script uses the community
|
||||
// `k6/x/sse` extension (github.com/phymbert/xk6-sse). Recent k6 releases
|
||||
// resolve `k6/x/sse` automatically; otherwise build a binary with:
|
||||
// `k6/x/sse` extension (github.com/phymbert/xk6-sse), which is not in the
|
||||
// stock k6 binary. Build the pinned binary once:
|
||||
// make k6-build
|
||||
//
|
||||
// Run:
|
||||
// API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js
|
||||
// API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
|
||||
// make load
|
||||
//
|
||||
// Recorded metrics:
|
||||
// sse_connections_opened counter, expected >= SSE_VUS (120)
|
||||
// sse_connections_active gauge, peak should reach 120
|
||||
// sse_connections_closed counter, expected to equal connections_opened
|
||||
// sse_open_success rate of VUs that opened a stream
|
||||
// sse_event_latency p95 server-timestamp-to-client latency, ms
|
||||
// sse_scenario_latency same, scenario events only (if any occur)
|
||||
@@ -24,7 +24,7 @@
|
||||
import http from 'k6/http';
|
||||
import sse from 'k6/x/sse';
|
||||
import { check } from 'k6';
|
||||
import { Counter, Gauge, Rate, Trend } from 'k6/metrics';
|
||||
import { Counter, Rate, Trend } from 'k6/metrics';
|
||||
|
||||
const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, '');
|
||||
const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`;
|
||||
@@ -33,7 +33,7 @@ const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20));
|
||||
const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m';
|
||||
|
||||
const connectionsOpened = new Counter('sse_connections_opened');
|
||||
const connectionsActive = new Gauge('sse_connections_active');
|
||||
const connectionsClosed = new Counter('sse_connections_closed');
|
||||
const openSuccess = new Rate('sse_open_success');
|
||||
const connectionErrors = new Counter('sse_connection_errors');
|
||||
const eventsReceived = new Counter('sse_events_received');
|
||||
@@ -73,23 +73,10 @@ export function setup() {
|
||||
|
||||
export default function () {
|
||||
let opened = false;
|
||||
let active = false;
|
||||
let closedByUs = false;
|
||||
let firstScenario = true;
|
||||
let pings = 0;
|
||||
|
||||
const markActive = () => {
|
||||
if (!active) {
|
||||
active = true;
|
||||
connectionsActive.add(1);
|
||||
}
|
||||
};
|
||||
const markInactive = () => {
|
||||
if (active) {
|
||||
active = false;
|
||||
connectionsActive.add(-1);
|
||||
}
|
||||
};
|
||||
|
||||
const params = {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
@@ -104,7 +91,6 @@ export default function () {
|
||||
const res = sse.open(STREAM_URL, params, function (client) {
|
||||
client.on('open', function () {
|
||||
opened = true;
|
||||
markActive();
|
||||
connectionsOpened.add(1);
|
||||
});
|
||||
|
||||
@@ -123,9 +109,18 @@ export default function () {
|
||||
}
|
||||
|
||||
// The contract puts the emit time in `t` (ping) or `at` (scenario).
|
||||
// The first scenario event on connect replays the current state and
|
||||
// carries the original state-change timestamp, not the send time, so
|
||||
// exclude it per VU. Only scenario changes broadcast during the run
|
||||
// are valid samples.
|
||||
const initialScenario = name === 'scenario' && firstScenario;
|
||||
if (name === 'scenario') {
|
||||
firstScenario = false;
|
||||
}
|
||||
|
||||
// Latency is server timestamp -> client receipt, in milliseconds.
|
||||
const stamp = payload && (payload.t || payload.at);
|
||||
if (stamp) {
|
||||
if (stamp && !initialScenario) {
|
||||
const emitted = Date.parse(stamp);
|
||||
if (!Number.isNaN(emitted)) {
|
||||
const lag = now - emitted;
|
||||
@@ -142,7 +137,6 @@ export default function () {
|
||||
pings += 1;
|
||||
if (pings >= HOLD_PINGS && !closedByUs) {
|
||||
closedByUs = true;
|
||||
markInactive();
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
@@ -153,10 +147,12 @@ export default function () {
|
||||
connectionErrors.add(1);
|
||||
console.error(`SSE error: ${e && e.error ? e.error() : e}`);
|
||||
}
|
||||
markInactive();
|
||||
});
|
||||
});
|
||||
|
||||
if (opened) {
|
||||
connectionsClosed.add(1);
|
||||
}
|
||||
check(res, {
|
||||
'sse status 200': (r) => r && r.status === 200,
|
||||
'sse content-type': (r) =>
|
||||
@@ -168,5 +164,4 @@ export default function () {
|
||||
connectionErrors.add(1);
|
||||
}
|
||||
openSuccess.add(opened);
|
||||
markInactive();
|
||||
}
|
||||
Executable
+470
@@ -0,0 +1,470 @@
|
||||
#!/usr/bin/env bash
|
||||
# Read-only Gate 1 readiness check.
|
||||
#
|
||||
# It NEVER applies, creates, patches, deletes, scales, or restarts anything.
|
||||
# Allowed calls: kubectl config/get/auth can-i/apply --dry-run=client, curl.
|
||||
#
|
||||
# Usage (normally through make):
|
||||
# export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml
|
||||
# make gate1-check
|
||||
# make gate1-check CHECK_PUBLIC=1 # also curl the public URLs
|
||||
# make gate1-check ALLOW_PENDING=1 # exit 0 while human secrets are pending
|
||||
#
|
||||
# Exit codes: 0 ready, 1 hard failure (permissions/manifests/cluster), 2 only
|
||||
# human prerequisites are missing (Secrets, Deployments, public URLs).
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
NS="${NS:-fch26-onemanarmy}"
|
||||
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"
|
||||
REGISTRY="${REGISTRY:-git.noelsabia.dev}"
|
||||
OWNER="${OWNER:-noelsabia}"
|
||||
TAG="${TAG:-gate1}"
|
||||
CHECK_PUBLIC="${CHECK_PUBLIC:-0}"
|
||||
ALLOW_PENDING="${ALLOW_PENDING:-0}"
|
||||
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
|
||||
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-15}"
|
||||
|
||||
hard=0
|
||||
pending=0
|
||||
warnings=0
|
||||
|
||||
ok() { printf '[ok] %s\n' "$*"; }
|
||||
bad() { printf '[FAIL] %s\n' "$*"; hard=$((hard + 1)); }
|
||||
missing() { printf '[pending] %s\n' "$*"; pending=$((pending + 1)); }
|
||||
warn() { printf '[warn] %s\n' "$*"; warnings=$((warnings + 1)); }
|
||||
section() { printf '\n== %s\n' "$*"; }
|
||||
|
||||
finish() {
|
||||
printf '\n== Gate 1 readiness summary\n'
|
||||
printf 'hard failures: %d\n' "$hard"
|
||||
printf 'pending prerequisites: %d\n' "$pending"
|
||||
printf 'warnings: %d\n' "$warnings"
|
||||
if ((hard > 0)); then
|
||||
printf 'RESULT: NOT READY (fix the failures above)\n'
|
||||
exit 1
|
||||
fi
|
||||
if ((pending > 0)); then
|
||||
if [[ "$ALLOW_PENDING" == "1" ]]; then
|
||||
printf 'RESULT: READY except for pending prerequisites (ALLOW_PENDING=1)\n'
|
||||
exit 0
|
||||
fi
|
||||
printf 'RESULT: NOT READY (prerequisites above are still pending)\n'
|
||||
exit 2
|
||||
fi
|
||||
printf 'RESULT: READY for Gate 1\n'
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap finish EXIT
|
||||
|
||||
printf 'Gate 1 readiness (read-only; no cluster mutation)\n'
|
||||
printf 'namespace=%s context=%s tag=%s\n' "$NS" "$CONTEXT" "$TAG"
|
||||
|
||||
section "kubeconfig and cluster"
|
||||
if [[ -z "${KUBECONFIG:-}" ]]; then
|
||||
bad "KUBECONFIG is not set"
|
||||
printf ' export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml\n'
|
||||
else
|
||||
if [[ ! -f "$KUBECONFIG" ]]; then
|
||||
bad "KUBECONFIG does not point to a file: $KUBECONFIG"
|
||||
else
|
||||
mode="$(stat -c '%a' "$KUBECONFIG" 2>/dev/null || echo unknown)"
|
||||
if [[ "$mode" != "600" && "$mode" != "400" ]]; then
|
||||
warn "kubeconfig mode is $mode; run: chmod 600 \"$KUBECONFIG\""
|
||||
else
|
||||
ok "kubeconfig mode $mode"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! command -v kubectl >/dev/null 2>&1; then
|
||||
bad "kubectl not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! kubectl config get-contexts -o name 2>/dev/null | grep -Fxq "$CONTEXT"; then
|
||||
bad "context '$CONTEXT' not found in KUBECONFIG"
|
||||
exit 1
|
||||
fi
|
||||
ok "context '$CONTEXT' exists"
|
||||
|
||||
KC=(kubectl --context "$CONTEXT" --namespace "$NS")
|
||||
|
||||
if "${KC[@]}" get --raw=/readyz --request-timeout=5s >/dev/null 2>&1; then
|
||||
ok "API server reachable"
|
||||
else
|
||||
bad "API server is not reachable with context '$CONTEXT'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if "${KC[@]}" get deployments >/dev/null 2>&1; then
|
||||
ok "namespace '$NS' is readable"
|
||||
else
|
||||
bad "cannot read namespace '$NS'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
section "namespace permissions (expected values)"
|
||||
check_can_i() {
|
||||
local expected="$1" verb="$2" resource="$3" got
|
||||
got="$("${KC[@]}" auth can-i "$verb" "$resource" 2>/dev/null)"
|
||||
if [[ "$got" == "$expected" ]]; then
|
||||
ok "$verb $resource -> $got"
|
||||
else
|
||||
bad "$verb $resource -> ${got:-<error>} (expected $expected)"
|
||||
fi
|
||||
}
|
||||
check_can_i yes create deployments
|
||||
check_can_i yes create services
|
||||
check_can_i yes create configmaps
|
||||
check_can_i yes create secrets
|
||||
check_can_i yes create pods
|
||||
check_can_i no create ingresses
|
||||
check_can_i no create jobs
|
||||
check_can_i no create cronjobs
|
||||
check_can_i no create networkpolicies
|
||||
check_can_i no create roles
|
||||
check_can_i no create rolebindings
|
||||
|
||||
section "manifest guardrails"
|
||||
if manifest_out="$(python3 - "$ROOT/deploy/k8s" "$NS" "$TAG" "$REGISTRY" "$OWNER" <<'PY' 2>&1
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
root, ns, tag, registry, owner = sys.argv[1:6]
|
||||
errors = []
|
||||
warns = []
|
||||
docs = []
|
||||
|
||||
for path in sorted(glob.glob(os.path.join(root, "*.yaml"))):
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
try:
|
||||
loaded = [d for d in __import__("yaml").safe_load_all(handle) if d]
|
||||
except Exception as exc: # noqa: BLE001 - report the parse error verbatim
|
||||
errors.append(f"{os.path.basename(path)}: YAML parse error: {exc}")
|
||||
continue
|
||||
if not loaded:
|
||||
errors.append(f"{os.path.basename(path)}: no YAML documents")
|
||||
for doc in loaded:
|
||||
doc["__file__"] = os.path.basename(path)
|
||||
docs.append(doc)
|
||||
|
||||
if not docs:
|
||||
errors.append("deploy/k8s contains no manifests")
|
||||
|
||||
allowed = {"ConfigMap", "Deployment", "Service"}
|
||||
forbidden = {
|
||||
"Ingress", "Job", "CronJob", "NetworkPolicy", "Role", "RoleBinding",
|
||||
"ClusterRole", "ClusterRoleBinding", "CustomResourceDefinition",
|
||||
"Namespace", "PersistentVolume", "PersistentVolumeClaim", "StorageClass",
|
||||
"PodSecurityPolicy", "PriorityClass", "APIService",
|
||||
"MutatingWebhookConfiguration", "ValidatingWebhookConfiguration",
|
||||
}
|
||||
|
||||
objects = {}
|
||||
configmaps = {}
|
||||
services = {}
|
||||
deployments = {}
|
||||
|
||||
for doc in docs:
|
||||
where = f"{doc['__file__']}:{doc.get('kind', '?')}/{doc.get('metadata', {}).get('name', '?')}"
|
||||
kind = doc.get("kind")
|
||||
meta = doc.get("metadata") or {}
|
||||
name = meta.get("name")
|
||||
if not doc.get("apiVersion") or not kind or not name:
|
||||
errors.append(f"{where}: missing apiVersion, kind, or metadata.name")
|
||||
continue
|
||||
if kind not in allowed:
|
||||
errors.append(f"{where}: kind {kind} is not allowed (namespace-only: ConfigMap, Deployment, Service)")
|
||||
continue
|
||||
if kind in forbidden:
|
||||
errors.append(f"{where}: forbidden kind {kind}")
|
||||
if meta.get("namespace") != ns:
|
||||
errors.append(f"{where}: metadata.namespace is {meta.get('namespace')!r}, expected {ns!r}")
|
||||
key = (kind, name)
|
||||
if key in objects:
|
||||
errors.append(f"{where}: duplicate object")
|
||||
objects[key] = doc
|
||||
if kind == "ConfigMap":
|
||||
configmaps[name] = doc
|
||||
elif kind == "Service":
|
||||
services[name] = doc
|
||||
elif kind == "Deployment":
|
||||
deployments[name] = doc
|
||||
|
||||
for name, cm in configmaps.items():
|
||||
data = cm.get("data") or {}
|
||||
for key in data:
|
||||
if re.search(r"(?i)(token|password|secret|private[_-]?key|credential)", key):
|
||||
errors.append(f"ConfigMap/{name}: key {key!r} looks like a secret; use a Secret created by the human")
|
||||
|
||||
required_config = {
|
||||
"pulsecast-shared": {"PUBLIC_WEB_URL", "PUBLIC_API_URL", "API_INTERNAL_URL", "ML_INTERNAL_URL"},
|
||||
"web-config": {"PORT", "NUXT_PUBLIC_API_BASE", "NUXT_API_INTERNAL_BASE"},
|
||||
"api-config": {"PORT", "DATA_DIR", "FASTAPI_URL", "ML_TIMEOUT", "SSE_HEARTBEAT"},
|
||||
"ml-config": {
|
||||
"PORT", "PULSECAST_ROOT", "PULSECAST_ARTIFACTS", "PULSECAST_WEATHER",
|
||||
"PULSECAST_MODELS", "PULSECAST_UPLOADS",
|
||||
},
|
||||
}
|
||||
for name, keys in required_config.items():
|
||||
cm = configmaps.get(name)
|
||||
if cm is None:
|
||||
errors.append(f"missing ConfigMap/{name}")
|
||||
continue
|
||||
have = set((cm.get("data") or {}).keys())
|
||||
for key in sorted(keys - have):
|
||||
errors.append(f"ConfigMap/{name}: missing key {key}")
|
||||
|
||||
expected_ports = {"web": 3000, "api": 8080, "ml": 8000, "cloudflared": 2000}
|
||||
expected_uids = {"web": 10001, "api": 65532, "ml": 10001, "cloudflared": 65532}
|
||||
expected_images = {
|
||||
"web": f"{registry}/{owner}/pulsecast-web:{tag}",
|
||||
"api": f"{registry}/{owner}/pulsecast-api:{tag}",
|
||||
"ml": f"{registry}/{owner}/pulsecast-ml:{tag}",
|
||||
}
|
||||
gitea_images = {"web", "api", "ml"}
|
||||
|
||||
for name in ("web", "api", "ml", "cloudflared"):
|
||||
dep = deployments.get(name)
|
||||
if dep is None:
|
||||
errors.append(f"missing Deployment/{name}")
|
||||
continue
|
||||
spec = dep.get("spec") or {}
|
||||
if spec.get("replicas") != 1:
|
||||
errors.append(f"Deployment/{name}: replicas should be 1")
|
||||
template = (spec.get("template") or {})
|
||||
pod = template.get("spec") or {}
|
||||
if pod.get("automountServiceAccountToken") is not False:
|
||||
errors.append(f"Deployment/{name}: automountServiceAccountToken must be false")
|
||||
pod_security = pod.get("securityContext") or {}
|
||||
if (pod_security.get("seccompProfile") or {}).get("type") != "RuntimeDefault":
|
||||
errors.append(f"Deployment/{name}: seccompProfile must be RuntimeDefault")
|
||||
for field in ("runAsUser", "runAsGroup"):
|
||||
value = pod_security.get(field)
|
||||
if not isinstance(value, int) or value <= 0:
|
||||
errors.append(f"Deployment/{name}: {field} must be a positive numeric id (SKE runAsNonRoot caveat)")
|
||||
elif value != expected_uids[name]:
|
||||
errors.append(f"Deployment/{name}: {field} is {value}, expected {expected_uids[name]}")
|
||||
if pod_security.get("runAsNonRoot") is True and pod_security.get("runAsUser") == 0:
|
||||
errors.append(f"Deployment/{name}: runAsUser 0 conflicts with runAsNonRoot")
|
||||
for field in ("hostNetwork", "hostPID", "hostIPC"):
|
||||
if pod.get(field):
|
||||
errors.append(f"Deployment/{name}: {field} is not allowed")
|
||||
if name in gitea_images:
|
||||
pulls = pod.get("imagePullSecrets") or []
|
||||
if not any(p.get("name") == "gitea-regcred" for p in pulls):
|
||||
errors.append(f"Deployment/{name}: imagePullSecrets must include gitea-regcred")
|
||||
containers = pod.get("containers") or []
|
||||
if len(containers) != 1:
|
||||
errors.append(f"Deployment/{name}: expected exactly one container")
|
||||
continue
|
||||
container = containers[0]
|
||||
if container.get("name") != name:
|
||||
errors.append(f"Deployment/{name}: container name is {container.get('name')!r}")
|
||||
image = container.get("image") or ""
|
||||
if image.endswith(":latest"):
|
||||
errors.append(f"Deployment/{name}: image tag must be pinned, not :latest")
|
||||
if name in gitea_images and not image.startswith(f"{registry}/{owner}/"):
|
||||
errors.append(f"Deployment/{name}: image {image!r} is not under {registry}/{owner}/")
|
||||
if name in expected_images and image != expected_images[name]:
|
||||
warns.append(f"Deployment/{name}: image {image!r} does not match TAG={tag}")
|
||||
if name == "cloudflared" and not image.startswith("cloudflare/cloudflared:"):
|
||||
errors.append(f"Deployment/{name}: unexpected image {image!r}")
|
||||
if name in expected_ports:
|
||||
ports = container.get("ports") or []
|
||||
if not any(int(p.get("containerPort", -1)) == expected_ports[name] for p in ports):
|
||||
errors.append(f"Deployment/{name}: containerPort {expected_ports[name]} not found")
|
||||
for port in ports:
|
||||
if port.get("hostPort"):
|
||||
errors.append(f"Deployment/{name}: hostPort is not allowed")
|
||||
security = container.get("securityContext") or {}
|
||||
if security.get("runAsNonRoot") is not True:
|
||||
errors.append(f"Deployment/{name}: runAsNonRoot must be true")
|
||||
if security.get("allowPrivilegeEscalation") is not False:
|
||||
errors.append(f"Deployment/{name}: allowPrivilegeEscalation must be false")
|
||||
if security.get("readOnlyRootFilesystem") is not True:
|
||||
errors.append(f"Deployment/{name}: readOnlyRootFilesystem must be true")
|
||||
if security.get("privileged") is True:
|
||||
errors.append(f"Deployment/{name}: privileged is not allowed")
|
||||
if security.get("runAsUser") == 0:
|
||||
errors.append(f"Deployment/{name}: runAsUser 0 is not allowed")
|
||||
drops = ((security.get("capabilities") or {}).get("drop") or [])
|
||||
if "ALL" not in drops:
|
||||
errors.append(f"Deployment/{name}: capabilities.drop must include ALL")
|
||||
resources = container.get("resources") or {}
|
||||
requests = resources.get("requests") or {}
|
||||
limits = resources.get("limits") or {}
|
||||
for group_name, group in (("requests", requests), ("limits", limits)):
|
||||
for key in ("cpu", "memory"):
|
||||
if not group.get(key):
|
||||
errors.append(f"Deployment/{name}: resources.{group_name}.{key} is missing")
|
||||
mounts = container.get("volumeMounts") or []
|
||||
tmp_mounts = [m for m in mounts if m.get("mountPath") == "/tmp"]
|
||||
if not tmp_mounts:
|
||||
errors.append(f"Deployment/{name}: /tmp volumeMount is missing")
|
||||
else:
|
||||
volume_name = tmp_mounts[0].get("name")
|
||||
volumes = pod.get("volumes") or []
|
||||
if not any(v.get("name") == volume_name and "emptyDir" in v for v in volumes):
|
||||
errors.append(f"Deployment/{name}: /tmp must be backed by an emptyDir")
|
||||
for probe in ("startupProbe", "livenessProbe", "readinessProbe"):
|
||||
probe_spec = container.get(probe) or {}
|
||||
http = probe_spec.get("httpGet") or {}
|
||||
if not http.get("path") or not http.get("port"):
|
||||
errors.append(f"Deployment/{name}: {probe}.httpGet is missing")
|
||||
refs = []
|
||||
for source in container.get("envFrom") or []:
|
||||
if source.get("configMapRef"):
|
||||
refs.append(source["configMapRef"].get("name"))
|
||||
for env in container.get("env") or []:
|
||||
if env.get("valueFrom", {}).get("configMapKeyRef"):
|
||||
refs.append(env["valueFrom"]["configMapKeyRef"].get("name"))
|
||||
for ref in refs:
|
||||
if ref not in configmaps:
|
||||
errors.append(f"Deployment/{name}: references missing ConfigMap/{ref}")
|
||||
expected_secret_refs = {
|
||||
"api": {
|
||||
("control-auth", "CONTROL_USER"),
|
||||
("control-auth", "CONTROL_PASS"),
|
||||
("control-auth", "CONTROL_TOKEN"),
|
||||
},
|
||||
"ml": {("control-auth", "CONTROL_TOKEN")},
|
||||
"cloudflared": {("cloudflared", "TUNNEL_TOKEN")},
|
||||
}
|
||||
seen_secret_refs = set()
|
||||
for env in container.get("env") or []:
|
||||
secret_ref = env.get("valueFrom", {}).get("secretKeyRef")
|
||||
if not secret_ref:
|
||||
continue
|
||||
secret_name = secret_ref.get("name")
|
||||
secret_key = secret_ref.get("key")
|
||||
if secret_name not in ("gitea-regcred", "cloudflared", "control-auth"):
|
||||
errors.append(f"Deployment/{name}: unexpected Secret reference {secret_name!r}")
|
||||
seen_secret_refs.add((secret_name, secret_key))
|
||||
for required in expected_secret_refs.get(name, set()):
|
||||
if required not in seen_secret_refs:
|
||||
errors.append(f"Deployment/{name}: missing Secret reference {required[0]}/{required[1]}")
|
||||
|
||||
if "cloudflared" in deployments:
|
||||
args = ((deployments["cloudflared"].get("spec") or {}).get("template") or {}).get("spec", {}).get("containers", [{}])[0].get("args") or []
|
||||
for required in ("tunnel", "run", "--token", "$(TUNNEL_TOKEN)", "--no-autoupdate", "--protocol", "http2"):
|
||||
if required not in args:
|
||||
errors.append(f"Deployment/cloudflared: args must include {required!r}")
|
||||
|
||||
for name in ("web", "api", "ml"):
|
||||
svc = services.get(name)
|
||||
if svc is None:
|
||||
errors.append(f"missing Service/{name}")
|
||||
continue
|
||||
spec = svc.get("spec") or {}
|
||||
if spec.get("type") != "ClusterIP":
|
||||
errors.append(f"Service/{name}: type must be ClusterIP")
|
||||
for field in ("externalIPs", "loadBalancerIP", "loadBalancerSourceRanges"):
|
||||
if spec.get(field):
|
||||
errors.append(f"Service/{name}: {field} is not allowed")
|
||||
ports = spec.get("ports") or []
|
||||
if not any(int(p.get("port", -1)) == expected_ports[name] for p in ports):
|
||||
errors.append(f"Service/{name}: port {expected_ports[name]} not found")
|
||||
if any(p.get("nodePort") for p in ports):
|
||||
errors.append(f"Service/{name}: nodePort is not allowed")
|
||||
|
||||
kinds = {}
|
||||
for kind, _ in objects:
|
||||
kinds[kind] = kinds.get(kind, 0) + 1
|
||||
summary = ", ".join(f"{kind}={count}" for kind, count in sorted(kinds.items()))
|
||||
print(f"parsed {len(objects)} objects ({summary})")
|
||||
for message in warns:
|
||||
print(f"WARN: {message}")
|
||||
for message in errors:
|
||||
print(f"FAIL: {message}")
|
||||
sys.exit(1 if errors else 0)
|
||||
PY
|
||||
)"; then
|
||||
ok "static guardrails passed"
|
||||
printf '%s\n' "$manifest_out" | sed 's/^/ /'
|
||||
else
|
||||
bad "static guardrails failed"
|
||||
printf '%s\n' "$manifest_out" | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
section "kubectl client dry-run"
|
||||
if dry_run_out="$("${KC[@]}" apply --dry-run=client -f "$ROOT/deploy/k8s" 2>&1)"; then
|
||||
count="$(printf '%s\n' "$dry_run_out" | grep -c '(dry run)')"
|
||||
ok "kubectl apply --dry-run=client passed ($count resources)"
|
||||
else
|
||||
bad "kubectl apply --dry-run=client failed"
|
||||
printf '%s\n' "$dry_run_out" | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
section "human prerequisites: Secrets"
|
||||
check_secret_keys() {
|
||||
local secret="$1"
|
||||
shift
|
||||
if ! "${KC[@]}" get secret "$secret" >/dev/null 2>&1; then
|
||||
missing "Secret/$secret is missing (create it with docs/deploy.md; never commit it)"
|
||||
return
|
||||
fi
|
||||
ok "Secret/$secret exists"
|
||||
local have key
|
||||
have="$("${KC[@]}" get secret "$secret" -o go-template='{{range $k,$v := .data}}{{$k}}{{"\n"}}{{end}}' 2>/dev/null || true)"
|
||||
for key in "$@"; do
|
||||
if printf '%s\n' "$have" | grep -qx "$key"; then
|
||||
ok "Secret/$secret has key $key"
|
||||
else
|
||||
missing "Secret/$secret is missing key $key"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
check_secret_keys gitea-regcred .dockerconfigjson
|
||||
if "${KC[@]}" get secret gitea-regcred >/dev/null 2>&1; then
|
||||
secret_type="$("${KC[@]}" get secret gitea-regcred -o jsonpath='{.type}' 2>/dev/null || true)"
|
||||
if [[ "$secret_type" == "kubernetes.io/dockerconfigjson" ]]; then
|
||||
ok "Secret/gitea-regcred type is kubernetes.io/dockerconfigjson"
|
||||
else
|
||||
bad "Secret/gitea-regcred type is ${secret_type:-<unknown>}, expected kubernetes.io/dockerconfigjson"
|
||||
fi
|
||||
fi
|
||||
check_secret_keys cloudflared TUNNEL_TOKEN
|
||||
check_secret_keys control-auth CONTROL_USER CONTROL_PASS CONTROL_TOKEN
|
||||
|
||||
section "deployment status"
|
||||
for deployment in web api ml cloudflared; do
|
||||
if ! "${KC[@]}" get deployment "$deployment" >/dev/null 2>&1; then
|
||||
missing "Deployment/$deployment is not deployed yet"
|
||||
continue
|
||||
fi
|
||||
ready="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)"
|
||||
desired="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
|
||||
image="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true)"
|
||||
if [[ -n "$ready" && -n "$desired" ]] && ((ready >= desired)); then
|
||||
ok "Deployment/$deployment ready ($ready/$desired, $image)"
|
||||
else
|
||||
bad "Deployment/$deployment not ready (${ready:-0}/${desired:-?}, $image)"
|
||||
fi
|
||||
if [[ "$deployment" != "cloudflared" && "$image" != *":$TAG" ]]; then
|
||||
warn "Deployment/$deployment image tag is not :$TAG"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$CHECK_PUBLIC" == "1" ]]; then
|
||||
section "public HTTPS"
|
||||
for target in "web|$WEB_URL/" "api|$API_URL/api/healthz"; do
|
||||
name="${target%%|*}"
|
||||
url="${target#*|}"
|
||||
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "$CURL_MAX_TIME" "$url" 2>/dev/null || echo 000)"
|
||||
if [[ "$code" == "200" ]]; then
|
||||
ok "$name public URL -> HTTP 200 ($url)"
|
||||
else
|
||||
missing "$name public URL -> HTTP ${code:-000} ($url)"
|
||||
fi
|
||||
done
|
||||
else
|
||||
section "public HTTPS"
|
||||
printf '[skip] CHECK_PUBLIC=0; run make smoke (or gate1-check CHECK_PUBLIC=1) after deploy\n'
|
||||
fi
|
||||
Regular → Executable
File mode changed.
Reference in new issue
Block a user