feat: deployment manifests and Gate 1 runbook

This commit is contained in:
Noel Sabia committed 2026-11-14 04:23:43 +01:00
1 parent aeee20cb7f
commit 55dfe8b616
3 files changed
+402 -54

No files matched your search

+76 -53
View File
@@ -6,6 +6,8 @@ ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
NAMESPACE ?= fch26-onemanarmy
CONTEXT ?= fch26-onemanarmy-sa-context
KUBECTL ?= kubectl --context $(CONTEXT) --namespace $(NAMESPACE)
KUBECONFORM ?= kubeconform
PYTHON ?= python3
REGISTRY ?= git.noelsabia.dev
OWNER ?= noelsabia
@@ -14,17 +16,26 @@ WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG)
WEB_DIR ?= $(ROOT)/../web-nuxt
API_DIR ?= $(ROOT)/../api-go
ML_DIR ?= $(ROOT)/../ML-py
# Service checkouts live beside the main checkout or in per-task worktrees:
# Hackathon_2026/worktrees/<repo>/<branch>
# Override any *_DIR on the command line when the layout differs.
SERVICES_ROOT ?= $(if $(wildcard $(abspath $(ROOT)/../../..)/worktrees),$(abspath $(ROOT)/../../..),$(abspath $(ROOT)/..))
WEB_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/web-nuxt/*) $(wildcard $(SERVICES_ROOT)/web-nuxt)),$(SERVICES_ROOT)/web-nuxt)
API_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/api-go/*) $(wildcard $(SERVICES_ROOT)/api-go)),$(SERVICES_ROOT)/api-go)
ML_DIR ?= $(or $(firstword $(wildcard $(SERVICES_ROOT)/worktrees/ML-py/*) $(wildcard $(SERVICES_ROOT)/ML-py)),$(SERVICES_ROOT)/ML-py)
WEB_URL ?= https://pulse.noelsabia.dev
API_URL ?= https://api.pulse.noelsabia.dev
ML_URL ?=
LOAD_API_URL ?= $(API_URL)
SMOKE_SSE ?= 0
SKIP_ML ?= 0
SSE_VUS ?= 120
SSE_HOLD_PINGS ?= 20
SSE_MAX_DURATION ?= 6m
K6_IMAGE ?= grafana/k6:latest
K6 ?=
K6_BIN ?= $(ROOT)/k6/k6-sse
K6_SCRIPT := $(ROOT)/k6/sse.js
K6_RESULTS := $(ROOT)/k6/results
@@ -33,48 +44,57 @@ K6_RESULTS := $(ROOT)/k6/results
help: ## Show available targets
@awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
validate: ## Client-side dry-run of deploy/k8s (never applies)
@command -v kubectl >/dev/null 2>&1 || { echo "kubectl not found"; exit 1; }
kubectl apply --dry-run=client --validate=false -f $(ROOT)/deploy/k8s
@if command -v kubeconform >/dev/null 2>&1; then \
kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s; \
validate: ## Validate deploy/k8s read-only (schema when possible, never applies)
@set -e; \
if command -v "$(KUBECONFORM)" >/dev/null 2>&1; then \
echo "== kubeconform (strict, offline schemas)"; \
"$(KUBECONFORM)" -strict -summary -ignore-missing-schemas "$(ROOT)/deploy/k8s"; \
elif command -v kubectl >/dev/null 2>&1 && kubectl cluster-info >/dev/null 2>&1; then \
echo "== kubectl client dry-run (read-only)"; \
kubectl apply --dry-run=client -f "$(ROOT)/deploy/k8s"; \
else \
echo "kubeconform not installed; kubectl client dry-run is the schema check"; \
echo "== YAML syntax only: kubeconform is not installed and no cluster is reachable"; \
"$(PYTHON)" -c "import glob, yaml; docs=[d for f in sorted(glob.glob('$(ROOT)/deploy/k8s/*.yaml')) for d in yaml.safe_load_all(open(f)) if d]; assert all(isinstance(d, dict) and d.get('apiVersion') and d.get('kind') for d in docs), 'manifest missing apiVersion/kind'; print(f'{len(docs)} manifests parsed')"; \
echo "Install kubeconform for schema validation: go install github.com/yannh/kubeconform/cmd/kubeconform@latest"; \
fi
validate-kubeconform: ## Validate schemas with kubeconform if installed
kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s
validate-kubeconform: ## Validate schemas with kubeconform (must be installed)
@command -v "$(KUBECONFORM)" >/dev/null 2>&1 || { echo "kubeconform not found; go install github.com/yannh/kubeconform/cmd/kubeconform@latest"; exit 1; }
"$(KUBECONFORM)" -strict -summary -ignore-missing-schemas "$(ROOT)/deploy/k8s"
build: build-web build-api build-ml ## Build all three images locally
build-web: ## Build the web image
@test -f $(WEB_DIR)/Dockerfile || { echo "missing $(WEB_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
docker build -t $(WEB_IMAGE) $(WEB_DIR)
@test -n "$(WEB_DIR)" || { echo "WEB_DIR is empty"; exit 1; }
@test -f "$(WEB_DIR)/Dockerfile" || { echo "missing $(WEB_DIR)/Dockerfile (see docs/deploy.md, Dockerfile contract)"; exit 1; }
docker build -t "$(WEB_IMAGE)" "$(WEB_DIR)"
build-api: ## Build the api image
@test -f $(API_DIR)/Dockerfile || { echo "missing $(API_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
docker build -t $(API_IMAGE) $(API_DIR)
@test -n "$(API_DIR)" || { echo "API_DIR is empty"; exit 1; }
@test -f "$(API_DIR)/Dockerfile" || { echo "missing $(API_DIR)/Dockerfile (see docs/deploy.md, Dockerfile contract)"; exit 1; }
docker build -t "$(API_IMAGE)" "$(API_DIR)"
build-ml: ## Build the ml image
@test -f $(ML_DIR)/Dockerfile || { echo "missing $(ML_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
docker build -t $(ML_IMAGE) $(ML_DIR)
@test -n "$(ML_DIR)" || { echo "ML_DIR is empty"; exit 1; }
@test -f "$(ML_DIR)/Dockerfile" || { echo "missing $(ML_DIR)/Dockerfile (see docs/deploy.md, Dockerfile contract)"; exit 1; }
docker build -t "$(ML_IMAGE)" "$(ML_DIR)"
push: push-web push-api push-ml ## Push all three images to Gitea
push-web: ## Push the web image
docker push $(WEB_IMAGE)
docker push "$(WEB_IMAGE)"
push-api: ## Push the api image
docker push $(API_IMAGE)
docker push "$(API_IMAGE)"
push-ml: ## Push the ml image
docker push $(ML_IMAGE)
docker push "$(ML_IMAGE)"
set-image: ## Update live Deployments to TAG (guarded; human only)
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
$(KUBECTL) set image deployment/web web=$(WEB_IMAGE)
$(KUBECTL) set image deployment/api api=$(API_IMAGE)
$(KUBECTL) set image deployment/ml ml=$(ML_IMAGE)
$(KUBECTL) set image deployment/web web="$(WEB_IMAGE)"
$(KUBECTL) set image deployment/api api="$(API_IMAGE)"
$(KUBECTL) set image deployment/ml ml="$(ML_IMAGE)"
$(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s
deploy-check: ## Verify required Secrets exist (read-only)
@@ -84,15 +104,16 @@ deploy-check: ## Verify required Secrets exist (read-only)
done
@echo "required Secrets present"
deploy: deploy-check ## Apply manifests (guarded; human only)
deploy: ## Apply manifests (guarded; human only)
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
$(KUBECTL) apply -f $(ROOT)/deploy/k8s
@$(MAKE) --no-print-directory deploy-check
$(KUBECTL) apply -f "$(ROOT)/deploy/k8s"
$(KUBECTL) rollout status deployment/ml --timeout=180s
$(KUBECTL) rollout status deployment/api --timeout=180s
$(KUBECTL) rollout status deployment/web --timeout=180s
$(KUBECTL) rollout status deployment/cloudflared --timeout=120s
@echo
@echo "Deployed. Now run: make smoke"
@echo "Deployed. Now run: make smoke SMOKE_SSE=1"
status: ## Show deployments, pods, services, and secrets
$(KUBECTL) get deploy,pods,svc,secrets
@@ -101,37 +122,39 @@ logs: ## Follow cloudflared logs
$(KUBECTL) logs -f deployment/cloudflared
smoke: ## Public HTTPS + API + ML health checks
WEB_URL="$(WEB_URL)" API_URL="$(API_URL)" NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" \
$(ROOT)/scripts/smoke.sh
WEB_URL="$(WEB_URL)" API_URL="$(API_URL)" ML_URL="$(ML_URL)" \
NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" \
SMOKE_SSE="$(SMOKE_SSE)" SKIP_ML="$(SKIP_ML)" \
"$(ROOT)/scripts/smoke.sh"
smoke-local: ## Smoke the three local dev servers
WEB_URL="http://localhost:3000" API_URL="http://localhost:8080" \
ML_URL="http://localhost:8000" $(ROOT)/scripts/smoke.sh
ML_URL="http://localhost:8000" \
SMOKE_SSE="$(SMOKE_SSE)" SKIP_ML="$(SKIP_ML)" \
"$(ROOT)/scripts/smoke.sh"
load: ## Run the 120-client / 5-minute SSE load test
@mkdir -p $(K6_RESULTS)
@export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" \
SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \
stamp=$$(date +%Y%m%dT%H%M%S); \
if command -v k6 >/dev/null 2>&1; then \
echo "Using k6 from PATH"; \
k6 run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \
elif [ -x "$(ROOT)/k6/k6-sse" ]; then \
echo "Using $(ROOT)/k6/k6-sse"; \
"$(ROOT)/k6/k6-sse" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \
@mkdir -p "$(K6_RESULTS)"
@set -e; \
if [ -n "$(K6)" ]; then \
runner="$(K6)"; \
elif [ -x "$(K6_BIN)" ]; then \
runner="$(K6_BIN)"; \
elif command -v k6 >/dev/null 2>&1; then \
runner="$$(command -v k6)"; \
else \
echo "Using docker image $(K6_IMAGE)"; \
docker run --rm -i \
-v "$(ROOT)/k6:/scripts:ro" \
-v "$(K6_RESULTS):/results" \
-e API_URL -e SSE_VUS -e SSE_HOLD_PINGS -e SSE_MAX_DURATION \
$(K6_IMAGE) run --summary-export="/results/summary-$$stamp.json" /scripts/sse.js; \
fi
echo "No k6 binary. Run 'make k6-build' once (needs Go), then retry."; \
exit 1; \
fi; \
export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \
stamp=$$(date +%Y%m%dT%H%M%S); \
echo "Using $$runner"; \
"$$runner" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"
k6-build: ## Build a k6 binary with the SSE extension (k6/k6-sse)
@mkdir -p $(K6_RESULTS)
cd $(ROOT)/k6 && GOBIN=$(ROOT)/k6 go install go.k6.io/xk6/cmd/xk6@latest
cd $(ROOT)/k6 && $(ROOT)/k6/xk6 build --with github.com/phymbert/xk6-sse@latest --output $(ROOT)/k6/k6-sse
@command -v go >/dev/null 2>&1 || { echo "Go is required to build xk6"; exit 1; }
@cd "$(ROOT)/k6" && GOBIN="$(ROOT)/k6" go install go.k6.io/xk6/cmd/xk6@latest
@cd "$(ROOT)/k6" && "$(ROOT)/k6/xk6" build --with github.com/phymbert/xk6-sse@latest --output "$(ROOT)/k6/k6-sse"
@echo "Built $(ROOT)/k6/k6-sse"
secret-commands: ## Print the exact Secret creation commands (creates nothing)
@@ -148,6 +171,6 @@ secret-commands: ## Print the exact Secret creation commands (creates nothing)
@echo " --docker-password='<read-only package token>'"
clean: ## Remove local k6 build artifacts and results
rm -f $(ROOT)/k6/k6-sse $(ROOT)/k6/xk6
rm -rf $(K6_RESULTS)
@mkdir -p $(K6_RESULTS) && touch $(K6_RESULTS)/.gitkeep
rm -f "$(K6_BIN)" "$(ROOT)/k6/xk6"
rm -rf "$(K6_RESULTS)"
@mkdir -p "$(K6_RESULTS)" && touch "$(K6_RESULTS)/.gitkeep"
+1 -1
View File
@@ -42,7 +42,7 @@ metadata:
data:
PORT: "8080"
DATA_DIR: "/app/data"
ML_BASE_URL: "http://ml:8000"
FASTAPI_URL: "http://ml:8000"
ML_TIMEOUT: "1.5s"
SSE_HEARTBEAT: "15s"
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
+325
View File
@@ -0,0 +1,325 @@
# Deploy runbook — Gate 1
Target: three services (`web`, `api`, `ml`) plus one `cloudflared` Deployment in
namespace `fch26-onemanarmy`, reachable over public HTTPS, with Gitea image
pulls working.
**The human runs every cluster-mutating command in this document.** The infra
agent only writes and validates manifests. No Ingress, cert-manager,
external-dns, Jobs, CronJobs, CRDs, Roles, RoleBindings, NetworkPolicies, or
other cluster-scoped resources are used.
## Gate 1 checklist
Run top to bottom. Stop at the first failure and use
[Troubleshooting](#troubleshooting).
- [ ] 1. Kubeconfig exported, `600`, correct context, namespace reachable.
- [ ] 2. Gitea **read-only package token** created and `docker login` succeeds.
- [ ] 3. Hello-world image pushed and pulled by the cluster with `gitea-regcred`.
- [ ] 4. Secrets created: `gitea-regcred`, `cloudflared`, `control-auth`.
- [ ] 5. Real `web`/`api`/`ml` images built, tagged `gate1`, pushed to Gitea.
- [ ] 6. `make validate` passes locally.
- [ ] 7. Cloudflare Tunnel exists; both public hostnames configured.
- [ ] 8. `make deploy CONFIRM=yes` completes and all four Deployments are ready.
- [ ] 9. `make smoke SMOKE_SSE=1` passes.
- [ ] 10. `https://pulse.noelsabia.dev` opens on a phone over mobile data.
- [ ] 11. `https://api.pulse.noelsabia.dev/api/healthz` returns 200.
- [ ] 12. Record the deployed image tags and the time in the status note.
## 0. Prerequisites
The kubeconfig is **not in this repo**. It lives outside git:
```bash
export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarmy-sa-kubeconfig.yaml"
export NS=fch26-onemanarmy
export CTX=fch26-onemanarmy-sa-context
export REGISTRY=git.noelsabia.dev
export OWNER=noelsabia
export TAG=gate1
chmod 600 "$KUBECONFIG"
```
Verify read-only first:
```bash
kubectl config current-context # fch26-onemanarmy-sa-context
kubectl -n "$NS" auth can-i create deployments # yes
kubectl -n "$NS" auth can-i create services # yes
kubectl -n "$NS" auth can-i create ingresses # no
kubectl -n "$NS" auth can-i create jobs # no
kubectl -n "$NS" get pods,svc,secrets
```
If any of the first three answers differ, stop and report; do not work around
permissions with cluster-scoped resources.
Local tools: `git`, `make`, `bash`, `curl`, `docker`, `kubectl`. Optional:
`kubeconform` (schema validation), `jq`, `k6` built with the SSE extension.
## 1. Gitea read-only package token
In Gitea: **Settings → Applications → Generate New Token** (or the package
token screen). Scope: read-only package access. Do not reuse the admin token.
```bash
read -rsp 'Gitea package token: ' GITEA_TOKEN; echo
printf '%s' "$GITEA_TOKEN" | docker login "$REGISTRY" -u "$OWNER" --password-stdin
```
A successful login prints `Login Succeeded`. Never put the token in a file,
a command that lands in shell history, or this repo.
## 2. Hello-world image pull (prove the registry path)
```bash
docker pull nginxinc/nginx-unprivileged:alpine
docker tag nginxinc/nginx-unprivileged:alpine "$REGISTRY/$OWNER/hello:gate1"
docker push "$REGISTRY/$OWNER/hello:gate1"
kubectl -n "$NS" create secret docker-registry gitea-regcred \
--docker-server="$REGISTRY" \
--docker-username="$OWNER" \
--docker-password="$GITEA_TOKEN"
kubectl -n "$NS" create deployment hello \
--image="$REGISTRY/$OWNER/hello:gate1"
kubectl -n "$NS" patch deployment hello --type=merge -p \
'{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"gitea-regcred"}]}}}}'
kubectl -n "$NS" rollout status deployment/hello --timeout=180s
kubectl -n "$NS" get pods -l app=hello
```
Pass: the pod is `Running` and `kubectl describe pod` shows the image pulled.
Clean up before the real deploy:
```bash
kubectl -n "$NS" delete deployment hello
```
If this fails, fix the token/secret before building the application images.
## 3. Create the remaining Secrets
Generate values locally; never commit them.
```bash
read -rsp 'Control password: ' CONTROL_PASS; echo
kubectl -n "$NS" create secret generic control-auth \
--from-literal=CONTROL_USER=control \
--from-literal=CONTROL_PASS="$CONTROL_PASS"
# Token from Cloudflare Zero Trust → Networks → Tunnels → your tunnel.
read -rsp 'Cloudflare tunnel token: ' TUNNEL_TOKEN; echo
kubectl -n "$NS" create secret generic cloudflared \
--from-literal=TUNNEL_TOKEN="$TUNNEL_TOKEN"
kubectl -n "$NS" get secrets
```
`make secret-commands` prints the same commands with placeholders.
## 4. Build and push the three images
The manifests expect:
| Service | Image | Port | Health paths |
|---|---|---|---|
| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `/` |
| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `/api/healthz`, `/api/readyz` |
| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `/healthz`, `/readyz` |
```bash
make build # builds all three from the service worktrees
make push # docker push to Gitea
```
`make build` auto-detects the service worktrees under
`/home/noel/Documents/Hackathon_2026/worktrees/`. Override when needed:
```bash
make build WEB_DIR=/path/to/web-nuxt API_DIR=/path/to/api-go ML_DIR=/path/to/ML-py
```
If `OWNER` or `TAG` differs from the manifest defaults, either update
`deploy/k8s/*.yaml` before `make deploy`, or deploy first and then run the
guarded image update:
```bash
make set-image CONFIRM=yes OWNER=<owner> TAG=<tag>
```
### Dockerfile contract (coordinate through the orchestrator)
Each service image must:
- listen on `0.0.0.0` at the port above;
- run as a non-root user (numeric UID, no shell required);
- write only to `/tmp` (the pod root filesystem is read-only);
- start with the environment in `deploy/k8s/00-config.yaml`;
- be self-contained: no runtime download of data, model, fonts, or packages.
`api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`,
`CONTROL_USER`, `CONTROL_PASS`, `SSE_HEARTBEAT`, and `ML_TIMEOUT`.
`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in or
provided at runtime; `ml` must find its model and artifacts under the paths its
Dockerfile defines. The env names in the manifests are the integration point:
change them here first if a service needs a different name.
## 5. Cloudflare Tunnel
1. Zero Trust → **Networks → Tunnels → Create a tunnel**.
2. Connector: **Cloudflared**. Name: `pulsecast-gate1`.
3. Copy the tunnel token (it is shown once). Store it in the `cloudflared`
Secret (step 3).
4. Add public hostnames:
| Public hostname | Service |
|---|---|
| `pulse.noelsabia.dev` | `http://web:3000` |
| `api.pulse.noelsabia.dev` | `http://api:8080` |
The hostnames must be in the Cloudflare zone for `noelsabia.dev`. The
service URL uses the Kubernetes Service name and port inside the namespace;
no Ingress is involved.
5. For the first smoke test before the app images exist, temporarily point
`pulse.noelsabia.dev` at `http://hello:8080`, then switch it to
`http://web:3000` after step 7.
DNS records for the public hostnames are created by Cloudflare; do not add
manual A/AAAA records for a tunnel hostname.
## 6. Validate before applying
```bash
make validate
```
This is read-only:
- `kubeconform -strict` validates the resource schemas if installed;
- otherwise, if a cluster is reachable, `kubectl apply --dry-run=client`
validates against the API server;
- if neither is available it parses the YAML and prints a warning that schema
validation was skipped.
Never run `kubectl apply` / `delete` from an agent session. The commands below
are for the human.
## 7. Deploy
```bash
make deploy-check # read-only: required Secrets exist
make deploy CONFIRM=yes # applies deploy/k8s and waits for rollouts
```
`make deploy` fails closed: without `CONFIRM=yes` it changes nothing. It does
not create the namespace or secrets.
If the three service images are not available yet, deploy only the tunnel and
use the hello service to prove public HTTPS; return to this step when the real
images exist. Do not deploy a placeholder image under the real service names.
Check the result:
```bash
kubectl -n "$NS" get deploy,pods,svc
kubectl -n "$NS" get pods -o wide
kubectl -n "$NS" logs deployment/cloudflared --tail=50 | grep 'Registered tunnel connection'
```
Expected: `web`, `api`, `ml`, `cloudflared` each `1/1`; Services `web`, `api`,
`ml` are `ClusterIP`; `cloudflared` has no Service (it dials out).
## 8. Verify
Public checks (also runnable from a phone on mobile data):
```bash
curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/
curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz
curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz
curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}'
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20
```
The SSE output must start with `retry: 3000`, contain a `scenario` event
immediately, and show a `ping` within 20 s.
Full smoke test, including the ML service through a temporary
`kubectl port-forward`:
```bash
make smoke SMOKE_SSE=1
```
Local variant once the three dev servers are running:
```bash
make smoke-local
```
In-cluster connectivity (optional; creates and removes one short-lived pod):
```bash
kubectl -n "$NS" run gate1-netcheck --rm -it --restart=Never \
--image=curlimages/curl:latest --command -- sh -c \
'curl -fsS http://api:8080/api/healthz && \
curl -fsS http://ml:8000/healthz && \
curl -fsS http://web:3000/ >/dev/null && echo connectivity-ok'
```
TLS is verified by `make smoke` (`ssl_verify_result=0`). Certificates are
issued by Cloudflare at the edge; there is no cert-manager resource.
## 9. Record the Gate 1 result
Write down, in the status note or README:
- deploy time and commit hash;
- image tags actually deployed (`kubectl -n "$NS" get deploy -o jsonpath=...`);
- output of `make smoke SMOKE_SSE=1`;
- public URLs and the device/network used;
- any warning that remains.
## Troubleshooting
| Symptom | Check | Fix |
|---|---|---|
| `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod <pod>`; `kubectl -n "$NS" get secret gitea-regcred` | Token is read-only package scoped; image tag exists; secret is in the same namespace. |
| `CreateContainerConfigError` | `kubectl -n "$NS" describe pod <pod>` | A referenced Secret key is missing (`control-auth` or `cloudflared`). Recreate the Secret. |
| `cloudflared` CrashLoopBackOff | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. |
| Public 502 / 1033 | `kubectl -n "$NS" get endpoints web api`; cloudflared logs | Service has no ready endpoints, wrong port, or hostname route points at the wrong service. |
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `kubectl -n "$NS" exec deploy/api -- ls /app/data` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. |
| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds 120 s or the model path is wrong. Fix the image/path; startupProbe allows 120 s. |
| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing build-time API base or server error. Rebuild with `NUXT_PUBLIC_API_BASE`. |
| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering; the web client falls back to polling `/api/scenario` every 5 s. Keep the heartbeat at 15 s. |
| `make deploy-check` fails | `kubectl -n "$NS" get secrets` | Create the missing Secret; do not commit it. |
## Rollback
No cluster-scoped resources are created, so rollback is namespace-local:
```bash
kubectl -n "$NS" rollout undo deployment/web
kubectl -n "$NS" rollout undo deployment/api
kubectl -n "$NS" rollout undo deployment/ml
kubectl -n "$NS" rollout undo deployment/cloudflared
kubectl -n "$NS" rollout status deployment/web --timeout=180s
```
If the tunnel token or route is wrong, fix the Secret / Cloudflare dashboard
and restart `cloudflared`; do not create an Ingress as a workaround.
## Security notes
- Never commit the kubeconfig, tunnel token, control password, Gitea token, or
any `.env` file. `.gitignore` covers the usual filenames; check
`git status` before every commit.
- Rotate the Gitea package token and the control password after the event.
Rotate the service-account token after the event; it was shared in chat.
- The kubeconfig must stay at mode `600` and outside the repos.
- `make smoke` prints no secrets; do not add `set -x` to the scripts with a
live `KUBECONFIG`.