From 55dfe8b616840cc9121a964b316d3e1ed63b32a0 Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 04:23:43 +0100 Subject: [PATCH] feat: deployment manifests and Gate 1 runbook --- Makefile | 129 ++++++++------- deploy/k8s/00-config.yaml | 2 +- docs/deploy.md | 325 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 402 insertions(+), 54 deletions(-) create mode 100644 docs/deploy.md diff --git a/Makefile b/Makefile index 04a4f46..c00cd97 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) NAMESPACE ?= fch26-onemanarmy CONTEXT ?= fch26-onemanarmy-sa-context KUBECTL ?= kubectl --context $(CONTEXT) --namespace $(NAMESPACE) +KUBECONFORM ?= kubeconform +PYTHON ?= python3 REGISTRY ?= git.noelsabia.dev OWNER ?= noelsabia @@ -14,17 +16,26 @@ WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG) API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG) ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG) -WEB_DIR ?= $(ROOT)/../web-nuxt -API_DIR ?= $(ROOT)/../api-go -ML_DIR ?= $(ROOT)/../ML-py +# Service checkouts live beside the main checkout or in per-task worktrees: +# Hackathon_2026/worktrees// +# Override any *_DIR on the command line when the layout differs. +SERVICES_ROOT ?= $(if $(wildcard $(abspath $(ROOT)/../../..)/worktrees),$(abspath $(ROOT)/../../..),$(abspath $(ROOT)/..)) +WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/web-nuxt/*) $(wildcard $(SERVICES_ROOT)/web-nuxt)),$(SERVICES_ROOT)/web-nuxt) +API_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/api-go/*) $(wildcard $(SERVICES_ROOT)/api-go)),$(SERVICES_ROOT)/api-go) +ML_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/ML-py/*) $(wildcard $(SERVICES_ROOT)/ML-py)),$(SERVICES_ROOT)/ML-py) WEB_URL ?= https://pulse.noelsabia.dev API_URL ?= https://api.pulse.noelsabia.dev +ML_URL ?= LOAD_API_URL ?= $(API_URL) +SMOKE_SSE ?= 0 +SKIP_ML ?= 0 + SSE_VUS ?= 120 SSE_HOLD_PINGS ?= 20 SSE_MAX_DURATION ?= 6m -K6_IMAGE ?= grafana/k6:latest +K6 ?= +K6_BIN ?= $(ROOT)/k6/k6-sse K6_SCRIPT := $(ROOT)/k6/sse.js K6_RESULTS := $(ROOT)/k6/results @@ -33,48 +44,57 @@ K6_RESULTS := $(ROOT)/k6/results help: ## Show available targets @awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST) -validate: ## Client-side dry-run of deploy/k8s (never applies) - @command -v kubectl >/dev/null 2>&1 || { echo "kubectl not found"; exit 1; } - kubectl apply --dry-run=client --validate=false -f $(ROOT)/deploy/k8s - @if command -v kubeconform >/dev/null 2>&1; then \ - kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s; \ +validate: ## Validate deploy/k8s read-only (schema when possible, never applies) + @set -e; \ + if command -v "$(KUBECONFORM)" >/dev/null 2>&1; then \ + echo "== kubeconform (strict, offline schemas)"; \ + "$(KUBECONFORM)" -strict -summary -ignore-missing-schemas "$(ROOT)/deploy/k8s"; \ + elif command -v kubectl >/dev/null 2>&1 && kubectl cluster-info >/dev/null 2>&1; then \ + echo "== kubectl client dry-run (read-only)"; \ + kubectl apply --dry-run=client -f "$(ROOT)/deploy/k8s"; \ else \ - echo "kubeconform not installed; kubectl client dry-run is the schema check"; \ + echo "== YAML syntax only: kubeconform is not installed and no cluster is reachable"; \ + "$(PYTHON)" -c "import glob, yaml; docs=[d for f in sorted(glob.glob('$(ROOT)/deploy/k8s/*.yaml')) for d in yaml.safe_load_all(open(f)) if d]; assert all(isinstance(d, dict) and d.get('apiVersion') and d.get('kind') for d in docs), 'manifest missing apiVersion/kind'; print(f'{len(docs)} manifests parsed')"; \ + echo "Install kubeconform for schema validation: go install github.com/yannh/kubeconform/cmd/kubeconform@latest"; \ fi -validate-kubeconform: ## Validate schemas with kubeconform if installed - kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s +validate-kubeconform: ## Validate schemas with kubeconform (must be installed) + @command -v "$(KUBECONFORM)" >/dev/null 2>&1 || { echo "kubeconform not found; go install github.com/yannh/kubeconform/cmd/kubeconform@latest"; exit 1; } + "$(KUBECONFORM)" -strict -summary -ignore-missing-schemas "$(ROOT)/deploy/k8s" build: build-web build-api build-ml ## Build all three images locally build-web: ## Build the web image - @test -f $(WEB_DIR)/Dockerfile || { echo "missing $(WEB_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; } - docker build -t $(WEB_IMAGE) $(WEB_DIR) + @test -n "$(WEB_DIR)" || { echo "WEB_DIR is empty"; exit 1; } + @test -f "$(WEB_DIR)/Dockerfile" || { echo "missing $(WEB_DIR)/Dockerfile (see docs/deploy.md, Dockerfile contract)"; exit 1; } + docker build -t "$(WEB_IMAGE)" "$(WEB_DIR)" build-api: ## Build the api image - @test -f $(API_DIR)/Dockerfile || { echo "missing $(API_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; } - docker build -t $(API_IMAGE) $(API_DIR) + @test -n "$(API_DIR)" || { echo "API_DIR is empty"; exit 1; } + @test -f "$(API_DIR)/Dockerfile" || { echo "missing $(API_DIR)/Dockerfile (see docs/deploy.md, Dockerfile contract)"; exit 1; } + docker build -t "$(API_IMAGE)" "$(API_DIR)" build-ml: ## Build the ml image - @test -f $(ML_DIR)/Dockerfile || { echo "missing $(ML_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; } - docker build -t $(ML_IMAGE) $(ML_DIR) + @test -n "$(ML_DIR)" || { echo "ML_DIR is empty"; exit 1; } + @test -f "$(ML_DIR)/Dockerfile" || { echo "missing $(ML_DIR)/Dockerfile (see docs/deploy.md, Dockerfile contract)"; exit 1; } + docker build -t "$(ML_IMAGE)" "$(ML_DIR)" push: push-web push-api push-ml ## Push all three images to Gitea push-web: ## Push the web image - docker push $(WEB_IMAGE) + docker push "$(WEB_IMAGE)" push-api: ## Push the api image - docker push $(API_IMAGE) + docker push "$(API_IMAGE)" push-ml: ## Push the ml image - docker push $(ML_IMAGE) + docker push "$(ML_IMAGE)" set-image: ## Update live Deployments to TAG (guarded; human only) @test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; } - $(KUBECTL) set image deployment/web web=$(WEB_IMAGE) - $(KUBECTL) set image deployment/api api=$(API_IMAGE) - $(KUBECTL) set image deployment/ml ml=$(ML_IMAGE) + $(KUBECTL) set image deployment/web web="$(WEB_IMAGE)" + $(KUBECTL) set image deployment/api api="$(API_IMAGE)" + $(KUBECTL) set image deployment/ml ml="$(ML_IMAGE)" $(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s deploy-check: ## Verify required Secrets exist (read-only) @@ -84,15 +104,16 @@ deploy-check: ## Verify required Secrets exist (read-only) done @echo "required Secrets present" -deploy: deploy-check ## Apply manifests (guarded; human only) +deploy: ## Apply manifests (guarded; human only) @test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; } - $(KUBECTL) apply -f $(ROOT)/deploy/k8s + @$(MAKE) --no-print-directory deploy-check + $(KUBECTL) apply -f "$(ROOT)/deploy/k8s" $(KUBECTL) rollout status deployment/ml --timeout=180s $(KUBECTL) rollout status deployment/api --timeout=180s $(KUBECTL) rollout status deployment/web --timeout=180s $(KUBECTL) rollout status deployment/cloudflared --timeout=120s @echo - @echo "Deployed. Now run: make smoke" + @echo "Deployed. Now run: make smoke SMOKE_SSE=1" status: ## Show deployments, pods, services, and secrets $(KUBECTL) get deploy,pods,svc,secrets @@ -101,37 +122,39 @@ logs: ## Follow cloudflared logs $(KUBECTL) logs -f deployment/cloudflared smoke: ## Public HTTPS + API + ML health checks - WEB_URL="$(WEB_URL)" API_URL="$(API_URL)" NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" \ - $(ROOT)/scripts/smoke.sh + WEB_URL="$(WEB_URL)" API_URL="$(API_URL)" ML_URL="$(ML_URL)" \ + NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" \ + SMOKE_SSE="$(SMOKE_SSE)" SKIP_ML="$(SKIP_ML)" \ + "$(ROOT)/scripts/smoke.sh" smoke-local: ## Smoke the three local dev servers WEB_URL="http://localhost:3000" API_URL="http://localhost:8080" \ - ML_URL="http://localhost:8000" $(ROOT)/scripts/smoke.sh + ML_URL="http://localhost:8000" \ + SMOKE_SSE="$(SMOKE_SSE)" SKIP_ML="$(SKIP_ML)" \ + "$(ROOT)/scripts/smoke.sh" load: ## Run the 120-client / 5-minute SSE load test - @mkdir -p $(K6_RESULTS) - @export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" \ - SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \ - stamp=$$(date +%Y%m%dT%H%M%S); \ - if command -v k6 >/dev/null 2>&1; then \ - echo "Using k6 from PATH"; \ - k6 run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \ - elif [ -x "$(ROOT)/k6/k6-sse" ]; then \ - echo "Using $(ROOT)/k6/k6-sse"; \ - "$(ROOT)/k6/k6-sse" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \ + @mkdir -p "$(K6_RESULTS)" + @set -e; \ + if [ -n "$(K6)" ]; then \ + runner="$(K6)"; \ + elif [ -x "$(K6_BIN)" ]; then \ + runner="$(K6_BIN)"; \ + elif command -v k6 >/dev/null 2>&1; then \ + runner="$$(command -v k6)"; \ else \ - echo "Using docker image $(K6_IMAGE)"; \ - docker run --rm -i \ - -v "$(ROOT)/k6:/scripts:ro" \ - -v "$(K6_RESULTS):/results" \ - -e API_URL -e SSE_VUS -e SSE_HOLD_PINGS -e SSE_MAX_DURATION \ - $(K6_IMAGE) run --summary-export="/results/summary-$$stamp.json" /scripts/sse.js; \ - fi + echo "No k6 binary. Run 'make k6-build' once (needs Go), then retry."; \ + exit 1; \ + fi; \ + export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \ + stamp=$$(date +%Y%m%dT%H%M%S); \ + echo "Using $$runner"; \ + "$$runner" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)" k6-build: ## Build a k6 binary with the SSE extension (k6/k6-sse) - @mkdir -p $(K6_RESULTS) - cd $(ROOT)/k6 && GOBIN=$(ROOT)/k6 go install go.k6.io/xk6/cmd/xk6@latest - cd $(ROOT)/k6 && $(ROOT)/k6/xk6 build --with github.com/phymbert/xk6-sse@latest --output $(ROOT)/k6/k6-sse + @command -v go >/dev/null 2>&1 || { echo "Go is required to build xk6"; exit 1; } + @cd "$(ROOT)/k6" && GOBIN="$(ROOT)/k6" go install go.k6.io/xk6/cmd/xk6@latest + @cd "$(ROOT)/k6" && "$(ROOT)/k6/xk6" build --with github.com/phymbert/xk6-sse@latest --output "$(ROOT)/k6/k6-sse" @echo "Built $(ROOT)/k6/k6-sse" secret-commands: ## Print the exact Secret creation commands (creates nothing) @@ -148,6 +171,6 @@ secret-commands: ## Print the exact Secret creation commands (creates nothing) @echo " --docker-password=''" clean: ## Remove local k6 build artifacts and results - rm -f $(ROOT)/k6/k6-sse $(ROOT)/k6/xk6 - rm -rf $(K6_RESULTS) - @mkdir -p $(K6_RESULTS) && touch $(K6_RESULTS)/.gitkeep + rm -f "$(K6_BIN)" "$(ROOT)/k6/xk6" + rm -rf "$(K6_RESULTS)" + @mkdir -p "$(K6_RESULTS)" && touch "$(K6_RESULTS)/.gitkeep" diff --git a/deploy/k8s/00-config.yaml b/deploy/k8s/00-config.yaml index a6c2ffc..7b65398 100644 --- a/deploy/k8s/00-config.yaml +++ b/deploy/k8s/00-config.yaml @@ -42,7 +42,7 @@ metadata: data: PORT: "8080" DATA_DIR: "/app/data" - ML_BASE_URL: "http://ml:8000" + FASTAPI_URL: "http://ml:8000" ML_TIMEOUT: "1.5s" SSE_HEARTBEAT: "15s" CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000" diff --git a/docs/deploy.md b/docs/deploy.md new file mode 100644 index 0000000..69cc6fa --- /dev/null +++ b/docs/deploy.md @@ -0,0 +1,325 @@ +# Deploy runbook — Gate 1 + +Target: three services (`web`, `api`, `ml`) plus one `cloudflared` Deployment in +namespace `fch26-onemanarmy`, reachable over public HTTPS, with Gitea image +pulls working. + +**The human runs every cluster-mutating command in this document.** The infra +agent only writes and validates manifests. No Ingress, cert-manager, +external-dns, Jobs, CronJobs, CRDs, Roles, RoleBindings, NetworkPolicies, or +other cluster-scoped resources are used. + +## Gate 1 checklist + +Run top to bottom. Stop at the first failure and use +[Troubleshooting](#troubleshooting). + +- [ ] 1. Kubeconfig exported, `600`, correct context, namespace reachable. +- [ ] 2. Gitea **read-only package token** created and `docker login` succeeds. +- [ ] 3. Hello-world image pushed and pulled by the cluster with `gitea-regcred`. +- [ ] 4. Secrets created: `gitea-regcred`, `cloudflared`, `control-auth`. +- [ ] 5. Real `web`/`api`/`ml` images built, tagged `gate1`, pushed to Gitea. +- [ ] 6. `make validate` passes locally. +- [ ] 7. Cloudflare Tunnel exists; both public hostnames configured. +- [ ] 8. `make deploy CONFIRM=yes` completes and all four Deployments are ready. +- [ ] 9. `make smoke SMOKE_SSE=1` passes. +- [ ] 10. `https://pulse.noelsabia.dev` opens on a phone over mobile data. +- [ ] 11. `https://api.pulse.noelsabia.dev/api/healthz` returns 200. +- [ ] 12. Record the deployed image tags and the time in the status note. + +## 0. Prerequisites + +The kubeconfig is **not in this repo**. It lives outside git: + +```bash +export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarmy-sa-kubeconfig.yaml" +export NS=fch26-onemanarmy +export CTX=fch26-onemanarmy-sa-context +export REGISTRY=git.noelsabia.dev +export OWNER=noelsabia +export TAG=gate1 + +chmod 600 "$KUBECONFIG" +``` + +Verify read-only first: + +```bash +kubectl config current-context # fch26-onemanarmy-sa-context +kubectl -n "$NS" auth can-i create deployments # yes +kubectl -n "$NS" auth can-i create services # yes +kubectl -n "$NS" auth can-i create ingresses # no +kubectl -n "$NS" auth can-i create jobs # no +kubectl -n "$NS" get pods,svc,secrets +``` + +If any of the first three answers differ, stop and report; do not work around +permissions with cluster-scoped resources. + +Local tools: `git`, `make`, `bash`, `curl`, `docker`, `kubectl`. Optional: +`kubeconform` (schema validation), `jq`, `k6` built with the SSE extension. + +## 1. Gitea read-only package token + +In Gitea: **Settings → Applications → Generate New Token** (or the package +token screen). Scope: read-only package access. Do not reuse the admin token. + +```bash +read -rsp 'Gitea package token: ' GITEA_TOKEN; echo +printf '%s' "$GITEA_TOKEN" | docker login "$REGISTRY" -u "$OWNER" --password-stdin +``` + +A successful login prints `Login Succeeded`. Never put the token in a file, +a command that lands in shell history, or this repo. + +## 2. Hello-world image pull (prove the registry path) + +```bash +docker pull nginxinc/nginx-unprivileged:alpine +docker tag nginxinc/nginx-unprivileged:alpine "$REGISTRY/$OWNER/hello:gate1" +docker push "$REGISTRY/$OWNER/hello:gate1" + +kubectl -n "$NS" create secret docker-registry gitea-regcred \ + --docker-server="$REGISTRY" \ + --docker-username="$OWNER" \ + --docker-password="$GITEA_TOKEN" + +kubectl -n "$NS" create deployment hello \ + --image="$REGISTRY/$OWNER/hello:gate1" +kubectl -n "$NS" patch deployment hello --type=merge -p \ + '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"gitea-regcred"}]}}}}' +kubectl -n "$NS" rollout status deployment/hello --timeout=180s +kubectl -n "$NS" get pods -l app=hello +``` + +Pass: the pod is `Running` and `kubectl describe pod` shows the image pulled. +Clean up before the real deploy: + +```bash +kubectl -n "$NS" delete deployment hello +``` + +If this fails, fix the token/secret before building the application images. + +## 3. Create the remaining Secrets + +Generate values locally; never commit them. + +```bash +read -rsp 'Control password: ' CONTROL_PASS; echo +kubectl -n "$NS" create secret generic control-auth \ + --from-literal=CONTROL_USER=control \ + --from-literal=CONTROL_PASS="$CONTROL_PASS" + +# Token from Cloudflare Zero Trust → Networks → Tunnels → your tunnel. +read -rsp 'Cloudflare tunnel token: ' TUNNEL_TOKEN; echo +kubectl -n "$NS" create secret generic cloudflared \ + --from-literal=TUNNEL_TOKEN="$TUNNEL_TOKEN" + +kubectl -n "$NS" get secrets +``` + +`make secret-commands` prints the same commands with placeholders. + +## 4. Build and push the three images + +The manifests expect: + +| Service | Image | Port | Health paths | +|---|---|---|---| +| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `/` | +| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `/api/healthz`, `/api/readyz` | +| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `/healthz`, `/readyz` | + +```bash +make build # builds all three from the service worktrees +make push # docker push to Gitea +``` + +`make build` auto-detects the service worktrees under +`/home/noel/Documents/Hackathon_2026/worktrees/`. Override when needed: + +```bash +make build WEB_DIR=/path/to/web-nuxt API_DIR=/path/to/api-go ML_DIR=/path/to/ML-py +``` + +If `OWNER` or `TAG` differs from the manifest defaults, either update +`deploy/k8s/*.yaml` before `make deploy`, or deploy first and then run the +guarded image update: + +```bash +make set-image CONFIRM=yes OWNER= TAG= +``` + +### Dockerfile contract (coordinate through the orchestrator) + +Each service image must: + +- listen on `0.0.0.0` at the port above; +- run as a non-root user (numeric UID, no shell required); +- write only to `/tmp` (the pod root filesystem is read-only); +- start with the environment in `deploy/k8s/00-config.yaml`; +- be self-contained: no runtime download of data, model, fonts, or packages. + +`api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`, +`CONTROL_USER`, `CONTROL_PASS`, `SSE_HEARTBEAT`, and `ML_TIMEOUT`. +`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in or +provided at runtime; `ml` must find its model and artifacts under the paths its +Dockerfile defines. The env names in the manifests are the integration point: +change them here first if a service needs a different name. + +## 5. Cloudflare Tunnel + +1. Zero Trust → **Networks → Tunnels → Create a tunnel**. +2. Connector: **Cloudflared**. Name: `pulsecast-gate1`. +3. Copy the tunnel token (it is shown once). Store it in the `cloudflared` + Secret (step 3). +4. Add public hostnames: + + | Public hostname | Service | + |---|---| + | `pulse.noelsabia.dev` | `http://web:3000` | + | `api.pulse.noelsabia.dev` | `http://api:8080` | + + The hostnames must be in the Cloudflare zone for `noelsabia.dev`. The + service URL uses the Kubernetes Service name and port inside the namespace; + no Ingress is involved. +5. For the first smoke test before the app images exist, temporarily point + `pulse.noelsabia.dev` at `http://hello:8080`, then switch it to + `http://web:3000` after step 7. + +DNS records for the public hostnames are created by Cloudflare; do not add +manual A/AAAA records for a tunnel hostname. + +## 6. Validate before applying + +```bash +make validate +``` + +This is read-only: + +- `kubeconform -strict` validates the resource schemas if installed; +- otherwise, if a cluster is reachable, `kubectl apply --dry-run=client` + validates against the API server; +- if neither is available it parses the YAML and prints a warning that schema + validation was skipped. + +Never run `kubectl apply` / `delete` from an agent session. The commands below +are for the human. + +## 7. Deploy + +```bash +make deploy-check # read-only: required Secrets exist +make deploy CONFIRM=yes # applies deploy/k8s and waits for rollouts +``` + +`make deploy` fails closed: without `CONFIRM=yes` it changes nothing. It does +not create the namespace or secrets. + +If the three service images are not available yet, deploy only the tunnel and +use the hello service to prove public HTTPS; return to this step when the real +images exist. Do not deploy a placeholder image under the real service names. + +Check the result: + +```bash +kubectl -n "$NS" get deploy,pods,svc +kubectl -n "$NS" get pods -o wide +kubectl -n "$NS" logs deployment/cloudflared --tail=50 | grep 'Registered tunnel connection' +``` + +Expected: `web`, `api`, `ml`, `cloudflared` each `1/1`; Services `web`, `api`, +`ml` are `ClusterIP`; `cloudflared` has no Service (it dials out). + +## 8. Verify + +Public checks (also runnable from a phone on mobile data): + +```bash +curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/ +curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz +curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz +curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}' +curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20 +``` + +The SSE output must start with `retry: 3000`, contain a `scenario` event +immediately, and show a `ping` within 20 s. + +Full smoke test, including the ML service through a temporary +`kubectl port-forward`: + +```bash +make smoke SMOKE_SSE=1 +``` + +Local variant once the three dev servers are running: + +```bash +make smoke-local +``` + +In-cluster connectivity (optional; creates and removes one short-lived pod): + +```bash +kubectl -n "$NS" run gate1-netcheck --rm -it --restart=Never \ + --image=curlimages/curl:latest --command -- sh -c \ + 'curl -fsS http://api:8080/api/healthz && \ + curl -fsS http://ml:8000/healthz && \ + curl -fsS http://web:3000/ >/dev/null && echo connectivity-ok' +``` + +TLS is verified by `make smoke` (`ssl_verify_result=0`). Certificates are +issued by Cloudflare at the edge; there is no cert-manager resource. + +## 9. Record the Gate 1 result + +Write down, in the status note or README: + +- deploy time and commit hash; +- image tags actually deployed (`kubectl -n "$NS" get deploy -o jsonpath=...`); +- output of `make smoke SMOKE_SSE=1`; +- public URLs and the device/network used; +- any warning that remains. + +## Troubleshooting + +| Symptom | Check | Fix | +|---|---|---| +| `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod `; `kubectl -n "$NS" get secret gitea-regcred` | Token is read-only package scoped; image tag exists; secret is in the same namespace. | +| `CreateContainerConfigError` | `kubectl -n "$NS" describe pod ` | A referenced Secret key is missing (`control-auth` or `cloudflared`). Recreate the Secret. | +| `cloudflared` CrashLoopBackOff | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. | +| Public 502 / 1033 | `kubectl -n "$NS" get endpoints web api`; cloudflared logs | Service has no ready endpoints, wrong port, or hostname route points at the wrong service. | +| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `kubectl -n "$NS" exec deploy/api -- ls /app/data` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. | +| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds 120 s or the model path is wrong. Fix the image/path; startupProbe allows 120 s. | +| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing build-time API base or server error. Rebuild with `NUXT_PUBLIC_API_BASE`. | +| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering; the web client falls back to polling `/api/scenario` every 5 s. Keep the heartbeat at 15 s. | +| `make deploy-check` fails | `kubectl -n "$NS" get secrets` | Create the missing Secret; do not commit it. | + +## Rollback + +No cluster-scoped resources are created, so rollback is namespace-local: + +```bash +kubectl -n "$NS" rollout undo deployment/web +kubectl -n "$NS" rollout undo deployment/api +kubectl -n "$NS" rollout undo deployment/ml +kubectl -n "$NS" rollout undo deployment/cloudflared +kubectl -n "$NS" rollout status deployment/web --timeout=180s +``` + +If the tunnel token or route is wrong, fix the Secret / Cloudflare dashboard +and restart `cloudflared`; do not create an Ingress as a workaround. + +## Security notes + +- Never commit the kubeconfig, tunnel token, control password, Gitea token, or + any `.env` file. `.gitignore` covers the usual filenames; check + `git status` before every commit. +- Rotate the Gitea package token and the control password after the event. + Rotate the service-account token after the event; it was shared in chat. +- The kubeconfig must stay at mode `600` and outside the repos. +- `make smoke` prints no secrets; do not add `set -x` to the scripts with a + live `KUBECONFIG`.