chore: bootstrap infra-k8 scaffold

This commit is contained in:
Noel Sabia committed 2026-11-14 04:16:22 +01:00
commit aeee20cb7f
14 files changed
+1256

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
# Local secrets and cluster credentials
*kubeconfig*
*.kubeconfig
.env
.env.*
secrets.local.yaml
deploy/secrets*.yaml
# k6 binaries and run output
k6/k6-sse
k6/xk6
k6/results/*
!k6/results/.gitkeep
# Editor / OS noise
.DS_Store
*.swp
+153
View File
@@ -0,0 +1,153 @@
SHELL := /usr/bin/env bash
.DEFAULT_GOAL := help
ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
NAMESPACE ?= fch26-onemanarmy
CONTEXT ?= fch26-onemanarmy-sa-context
KUBECTL ?= kubectl --context $(CONTEXT) --namespace $(NAMESPACE)
REGISTRY ?= git.noelsabia.dev
OWNER ?= noelsabia
TAG ?= gate1
WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG)
WEB_DIR ?= $(ROOT)/../web-nuxt
API_DIR ?= $(ROOT)/../api-go
ML_DIR ?= $(ROOT)/../ML-py
WEB_URL ?= https://pulse.noelsabia.dev
API_URL ?= https://api.pulse.noelsabia.dev
LOAD_API_URL ?= $(API_URL)
SSE_VUS ?= 120
SSE_HOLD_PINGS ?= 20
SSE_MAX_DURATION ?= 6m
K6_IMAGE ?= grafana/k6:latest
K6_SCRIPT := $(ROOT)/k6/sse.js
K6_RESULTS := $(ROOT)/k6/results
.PHONY: help validate validate-kubeconform build build-web build-api build-ml push push-web push-api push-ml set-image deploy deploy-check status logs smoke smoke-local load k6-build secret-commands clean
help: ## Show available targets
@awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
validate: ## Client-side dry-run of deploy/k8s (never applies)
@command -v kubectl >/dev/null 2>&1 || { echo "kubectl not found"; exit 1; }
kubectl apply --dry-run=client --validate=false -f $(ROOT)/deploy/k8s
@if command -v kubeconform >/dev/null 2>&1; then \
kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s; \
else \
echo "kubeconform not installed; kubectl client dry-run is the schema check"; \
fi
validate-kubeconform: ## Validate schemas with kubeconform if installed
kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s
build: build-web build-api build-ml ## Build all three images locally
build-web: ## Build the web image
@test -f $(WEB_DIR)/Dockerfile || { echo "missing $(WEB_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
docker build -t $(WEB_IMAGE) $(WEB_DIR)
build-api: ## Build the api image
@test -f $(API_DIR)/Dockerfile || { echo "missing $(API_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
docker build -t $(API_IMAGE) $(API_DIR)
build-ml: ## Build the ml image
@test -f $(ML_DIR)/Dockerfile || { echo "missing $(ML_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
docker build -t $(ML_IMAGE) $(ML_DIR)
push: push-web push-api push-ml ## Push all three images to Gitea
push-web: ## Push the web image
docker push $(WEB_IMAGE)
push-api: ## Push the api image
docker push $(API_IMAGE)
push-ml: ## Push the ml image
docker push $(ML_IMAGE)
set-image: ## Update live Deployments to TAG (guarded; human only)
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
$(KUBECTL) set image deployment/web web=$(WEB_IMAGE)
$(KUBECTL) set image deployment/api api=$(API_IMAGE)
$(KUBECTL) set image deployment/ml ml=$(ML_IMAGE)
$(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s
deploy-check: ## Verify required Secrets exist (read-only)
@for s in gitea-regcred cloudflared control-auth; do \
$(KUBECTL) get secret "$$s" >/dev/null 2>&1 || { \
echo "Missing Secret $$s in $(NAMESPACE); see docs/deploy.md"; exit 1; }; \
done
@echo "required Secrets present"
deploy: deploy-check ## Apply manifests (guarded; human only)
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
$(KUBECTL) apply -f $(ROOT)/deploy/k8s
$(KUBECTL) rollout status deployment/ml --timeout=180s
$(KUBECTL) rollout status deployment/api --timeout=180s
$(KUBECTL) rollout status deployment/web --timeout=180s
$(KUBECTL) rollout status deployment/cloudflared --timeout=120s
@echo
@echo "Deployed. Now run: make smoke"
status: ## Show deployments, pods, services, and secrets
$(KUBECTL) get deploy,pods,svc,secrets
logs: ## Follow cloudflared logs
$(KUBECTL) logs -f deployment/cloudflared
smoke: ## Public HTTPS + API + ML health checks
WEB_URL="$(WEB_URL)" API_URL="$(API_URL)" NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" \
$(ROOT)/scripts/smoke.sh
smoke-local: ## Smoke the three local dev servers
WEB_URL="http://localhost:3000" API_URL="http://localhost:8080" \
ML_URL="http://localhost:8000" $(ROOT)/scripts/smoke.sh
load: ## Run the 120-client / 5-minute SSE load test
@mkdir -p $(K6_RESULTS)
@export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" \
SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \
stamp=$$(date +%Y%m%dT%H%M%S); \
if command -v k6 >/dev/null 2>&1; then \
echo "Using k6 from PATH"; \
k6 run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \
elif [ -x "$(ROOT)/k6/k6-sse" ]; then \
echo "Using $(ROOT)/k6/k6-sse"; \
"$(ROOT)/k6/k6-sse" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \
else \
echo "Using docker image $(K6_IMAGE)"; \
docker run --rm -i \
-v "$(ROOT)/k6:/scripts:ro" \
-v "$(K6_RESULTS):/results" \
-e API_URL -e SSE_VUS -e SSE_HOLD_PINGS -e SSE_MAX_DURATION \
$(K6_IMAGE) run --summary-export="/results/summary-$$stamp.json" /scripts/sse.js; \
fi
k6-build: ## Build a k6 binary with the SSE extension (k6/k6-sse)
@mkdir -p $(K6_RESULTS)
cd $(ROOT)/k6 && GOBIN=$(ROOT)/k6 go install go.k6.io/xk6/cmd/xk6@latest
cd $(ROOT)/k6 && $(ROOT)/k6/xk6 build --with github.com/phymbert/xk6-sse@latest --output $(ROOT)/k6/k6-sse
@echo "Built $(ROOT)/k6/k6-sse"
secret-commands: ## Print the exact Secret creation commands (creates nothing)
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic cloudflared \\"
@echo " --from-literal=TUNNEL_TOKEN='<cloudflare tunnel token>'"
@echo
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic control-auth \\"
@echo " --from-literal=CONTROL_USER='<control user>' \\"
@echo " --from-literal=CONTROL_PASS='<strong password>'"
@echo
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret docker-registry gitea-regcred \\"
@echo " --docker-server=$(REGISTRY) \\"
@echo " --docker-username='<gitea user>' \\"
@echo " --docker-password='<read-only package token>'"
clean: ## Remove local k6 build artifacts and results
rm -f $(ROOT)/k6/k6-sse $(ROOT)/k6/xk6
rm -rf $(K6_RESULTS)
@mkdir -p $(K6_RESULTS) && touch $(K6_RESULTS)/.gitkeep
+48
View File
@@ -0,0 +1,48 @@
# PulseCast infra/QA
Namespace-only Kubernetes manifests, smoke tests, and the k6 SSE load test for
PulseCast. This repo never applies cluster changes by itself: the human runs the
commands in `docs/deploy.md`.
## Layout
```text
deploy/k8s/ web, api, ml + cloudflared Deployments and Services, ConfigMaps
deploy/examples/ example Secret shapes (placeholders only, never applied)
scripts/smoke.sh public + in-cluster smoke checks
k6/sse.js 120 concurrent SSE clients for 5 minutes
docs/deploy.md Gate 1 runbook and exact human checklist
docs/load-test.md load-test procedure and how to record results
docs/failure-drills.md FastAPI-down, tunnel-down, pod-restart drills
Makefile validate, build, push, deploy, smoke, load helpers
```
## Quick start
```bash
export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml # chmod 600
make validate # client-side dry run only, never applies
make deploy CONFIRM=yes # human only; requires the three Secrets first
make smoke # public HTTPS + API + ML health checks
make load # k6 SSE load test (needs the k6 SSE extension)
```
Read `docs/deploy.md` before the first deploy. It contains the Gitea token,
`docker login`, hello-world push, `gitea-regcred`, Cloudflare Tunnel, public
hostname, and verification steps.
## Git workflow
`main` is integration. New work happens on a `feat/<topic>` branch in its own
worktree. Before every push: `git fetch origin && git merge origin/main`.
Never force-push `main`. The initial scaffold is the one bootstrap exception
committed directly to `main`.
## Safety rules
- No Ingress, cert-manager, external-dns, Jobs, CronJobs, CRDs, Roles,
RoleBindings, NetworkPolicies, or any cluster-scoped resource.
- Exposure only through the `cloudflared` Deployment.
- No secrets, tokens, kubeconfig, or raw CSV in git.
- The human applies cluster mutations. `make deploy` refuses to run without
`CONFIRM=yes`.
+43
View File
@@ -0,0 +1,43 @@
# EXAMPLE ONLY — do not apply this file.
# It exists so the manifest shapes are reviewable. Create the real secrets with
# the commands in docs/deploy.md; never commit real tokens or credentials.
#
# kubectl -n fch26-onemanarmy create secret generic cloudflared \
# --from-literal=TUNNEL_TOKEN='<cloudflare tunnel token>'
#
# kubectl -n fch26-onemanarmy create secret generic control-auth \
# --from-literal=CONTROL_USER='<control user>' \
# --from-literal=CONTROL_PASS='<strong password>'
#
# kubectl -n fch26-onemanarmy create secret docker-registry gitea-regcred \
# --docker-server=git.noelsabia.dev \
# --docker-username='<gitea user>' \
# --docker-password='<read-only package token>'
apiVersion: v1
kind: Secret
metadata:
name: cloudflared
namespace: fch26-onemanarmy
type: Opaque
stringData:
TUNNEL_TOKEN: "REPLACE_ME"
---
apiVersion: v1
kind: Secret
metadata:
name: control-auth
namespace: fch26-onemanarmy
type: Opaque
stringData:
CONTROL_USER: "REPLACE_ME"
CONTROL_PASS: "REPLACE_ME"
---
apiVersion: v1
kind: Secret
metadata:
name: gitea-regcred
namespace: fch26-onemanarmy
type: kubernetes.io/dockerconfigjson
stringData:
.dockerconfigjson: |
{"auths":{"git.noelsabia.dev":{"username":"REPLACE_ME","password":"REPLACE_ME","auth":"REPLACE_ME"}}}
+63
View File
@@ -0,0 +1,63 @@
# Non-secret configuration for PulseCast.
# Namespace-scoped only. Secrets are created by the human, never committed.
apiVersion: v1
kind: ConfigMap
metadata:
name: pulsecast-shared
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/part-of: pulsecast
data:
PUBLIC_WEB_URL: "https://pulse.noelsabia.dev"
PUBLIC_API_URL: "https://api.pulse.noelsabia.dev"
API_INTERNAL_URL: "http://api:8080"
ML_INTERNAL_URL: "http://ml:8000"
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: web-config
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: web
app.kubernetes.io/part-of: pulsecast
data:
NODE_ENV: "production"
HOST: "0.0.0.0"
PORT: "3000"
NUXT_TELEMETRY_DISABLED: "1"
NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev"
NUXT_API_INTERNAL_BASE: "http://api:8080"
NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: api-config
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: api
app.kubernetes.io/part-of: pulsecast
data:
PORT: "8080"
DATA_DIR: "/app/data"
ML_BASE_URL: "http://ml:8000"
ML_TIMEOUT: "1.5s"
SSE_HEARTBEAT: "15s"
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ml-config
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: ml
app.kubernetes.io/part-of: pulsecast
data:
PORT: "8000"
DATA_DIR: "/app/data"
MODEL_PATH: "/app/model/model.joblib"
PYTHONDONTWRITEBYTECODE: "1"
PYTHONUNBUFFERED: "1"
+107
View File
@@ -0,0 +1,107 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ml
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: ml
app.kubernetes.io/component: ml
app.kubernetes.io/part-of: pulsecast
spec:
replicas: 1
revisionHistoryLimit: 2
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: ml
template:
metadata:
labels:
app.kubernetes.io/name: ml
app.kubernetes.io/component: ml
app.kubernetes.io/part-of: pulsecast
spec:
automountServiceAccountToken: false
imagePullSecrets:
- name: gitea-regcred
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: ml
image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 8000
protocol: TCP
envFrom:
- configMapRef:
name: ml-config
- configMapRef:
name: pulsecast-shared
env:
- name: HOME
value: /tmp
- name: MPLCONFIGDIR
value: /tmp
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
securityContext:
runAsNonRoot: true
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
volumeMounts:
- name: tmp
mountPath: /tmp
startupProbe:
httpGet:
path: /healthz
port: http
periodSeconds: 2
failureThreshold: 60
livenessProbe:
httpGet:
path: /healthz
port: http
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 3
readinessProbe:
httpGet:
path: /readyz
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 2
terminationGracePeriodSeconds: 30
volumes:
- name: tmp
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: ml
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: ml
app.kubernetes.io/part-of: pulsecast
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: ml
ports:
- name: http
port: 8000
targetPort: http
protocol: TCP
+118
View File
@@ -0,0 +1,118 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: api
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: api
app.kubernetes.io/component: backend
app.kubernetes.io/part-of: pulsecast
spec:
# Scenario state and the SSE hub live in memory. One replica is intentional.
replicas: 1
revisionHistoryLimit: 2
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: api
template:
metadata:
labels:
app.kubernetes.io/name: api
app.kubernetes.io/component: backend
app.kubernetes.io/part-of: pulsecast
spec:
automountServiceAccountToken: false
imagePullSecrets:
- name: gitea-regcred
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: api
image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 8080
protocol: TCP
envFrom:
- configMapRef:
name: api-config
- configMapRef:
name: pulsecast-shared
env:
- name: HOME
value: /tmp
# Created by the human; see docs/deploy.md. Never commit real credentials.
- name: CONTROL_USER
valueFrom:
secretKeyRef:
name: control-auth
key: CONTROL_USER
- name: CONTROL_PASS
valueFrom:
secretKeyRef:
name: control-auth
key: CONTROL_PASS
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 256Mi
securityContext:
runAsNonRoot: true
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
volumeMounts:
- name: tmp
mountPath: /tmp
startupProbe:
httpGet:
path: /api/healthz
port: http
periodSeconds: 2
failureThreshold: 30
livenessProbe:
httpGet:
path: /api/healthz
port: http
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 3
readinessProbe:
httpGet:
path: /api/readyz
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 2
# SSE clients reconnect with retry: 3000; give them a moment on shutdown.
terminationGracePeriodSeconds: 20
volumes:
- name: tmp
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: api
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: api
app.kubernetes.io/part-of: pulsecast
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: api
ports:
- name: http
port: 8080
targetPort: http
protocol: TCP
+111
View File
@@ -0,0 +1,111 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: web
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: web
app.kubernetes.io/component: frontend
app.kubernetes.io/part-of: pulsecast
spec:
replicas: 1
revisionHistoryLimit: 2
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 0
maxSurge: 1
selector:
matchLabels:
app.kubernetes.io/name: web
template:
metadata:
labels:
app.kubernetes.io/name: web
app.kubernetes.io/component: frontend
app.kubernetes.io/part-of: pulsecast
spec:
automountServiceAccountToken: false
imagePullSecrets:
- name: gitea-regcred
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: web
# Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`.
image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 3000
protocol: TCP
envFrom:
- configMapRef:
name: web-config
- configMapRef:
name: pulsecast-shared
env:
- name: HOME
value: /tmp
# Nuxt/Nitro may use a temp dir; keep it on the writable emptyDir.
- name: NITRO_TEMP_DIR
value: /tmp
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
securityContext:
runAsNonRoot: true
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
volumeMounts:
- name: tmp
mountPath: /tmp
startupProbe:
httpGet:
path: /
port: http
periodSeconds: 2
failureThreshold: 30
livenessProbe:
httpGet:
path: /
port: http
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 3
readinessProbe:
httpGet:
path: /
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 2
volumes:
- name: tmp
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: web
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: web
app.kubernetes.io/part-of: pulsecast
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: web
ports:
- name: http
port: 3000
targetPort: http
protocol: TCP
+99
View File
@@ -0,0 +1,99 @@
# Cloudflare Tunnel is the only public exposure path.
# Routes are configured in the Cloudflare Zero Trust dashboard:
# pulse.noelsabia.dev -> http://web:3000
# api.pulse.noelsabia.dev -> http://api:8080
# The token lives in the `cloudflared` Secret, created by the human; never in git.
apiVersion: apps/v1
kind: Deployment
metadata:
name: cloudflared
namespace: fch26-onemanarmy
labels:
app.kubernetes.io/name: cloudflared
app.kubernetes.io/component: tunnel
app.kubernetes.io/part-of: pulsecast
spec:
replicas: 1
revisionHistoryLimit: 2
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: cloudflared
template:
metadata:
labels:
app.kubernetes.io/name: cloudflared
app.kubernetes.io/component: tunnel
app.kubernetes.io/part-of: pulsecast
spec:
automountServiceAccountToken: false
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: cloudflared
# Public Docker Hub image; no Gitea pull secret needed for this pod.
image: cloudflare/cloudflared:latest
imagePullPolicy: IfNotPresent
args:
- tunnel
- --no-autoupdate
- --metrics
- 0.0.0.0:2000
- run
- --token
- $(TUNNEL_TOKEN)
env:
- name: TUNNEL_TOKEN
valueFrom:
secretKeyRef:
name: cloudflared
key: TUNNEL_TOKEN
- name: HOME
value: /tmp
ports:
- name: metrics
containerPort: 2000
protocol: TCP
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
securityContext:
runAsNonRoot: true
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
volumeMounts:
- name: tmp
mountPath: /tmp
startupProbe:
httpGet:
path: /ready
port: metrics
periodSeconds: 2
failureThreshold: 30
livenessProbe:
httpGet:
path: /ready
port: metrics
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 3
readinessProbe:
httpGet:
path: /ready
port: metrics
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 2
terminationGracePeriodSeconds: 30
volumes:
- name: tmp
emptyDir: {}
+96
View File
@@ -0,0 +1,96 @@
# Failure drills
Run these **after** Gate 1 and outside the demo window. Every command is a
cluster mutation: the human runs it, not an agent. Record the result and the
recovery time. Keep `make smoke` and a `curl -N .../api/stream` open while you
drill.
## 1. FastAPI down (cached fallback)
Goal: the Go API keeps serving forecasts from baked JSON with `"cached": true`
and `/api/readyz` stays 200. FastAPI being down must not fail API readiness.
```bash
# Baseline
curl -sS https://api.pulse.noelsabia.dev/api/readyz
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached'
# Stop ML
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=0
# Verify fallback (allow up to 2 s for the ML timeout)
curl -sS https://api.pulse.noelsabia.dev/api/readyz
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}'
# Expected: readyz 200; cached true; points still present
# Recover
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=1
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/ml --timeout=180s
```
Pass: no 5xx from `/api/forecast`, `cached` flips to `true` within the 1.5 s
timeout budget, and the UI shows the cached-forecast banner. Fail: a 502/503 or
an empty forecast.
## 2. Tunnel down
Goal: detect a public outage quickly, restart `cloudflared`, and verify both
public hostnames recover. Local `kubectl port-forward` is a diagnostic only,
never the public demo path.
```bash
# Stop the tunnel
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/cloudflared --replicas=0
# Verify the outage from outside
curl -sS -o /dev/null -w '%{http_code}\n' https://pulse.noelsabia.dev
curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz
# Expected: 502/503/530-style edge error, not a valid 200
# Recover
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/cloudflared --replicas=1
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/cloudflared --timeout=120s
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy logs deployment/cloudflared --tail=50
# Look for: "Registered tunnel connection"
make smoke
```
If the tunnel pod is Running but the URL is still 502, check the dashboard
routes (`pulse` -> `http://web:3000`, `api.pulse` -> `http://api:8080`) and the
Secret token. A bad token keeps the pod restarting with an auth error. Rotating
the tunnel token means updating the Secret and restarting the Deployment.
## 3. Pod restart (API and SSE reconnect)
Goal: a single API pod restart causes a short 502 window, not a stuck UI. SSE
clients reconnect with `retry: 3000`; scenario state is in memory and resets to
the default, which is documented.
```bash
# Watch from two terminals:
# curl -sS -N https://api.pulse.noelsabia.dev/api/stream
# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz; sleep 1; done
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy delete pod -l app.kubernetes.io/name=api
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/api --timeout=180s
curl -sS https://api.pulse.noelsabia.dev/api/healthz
curl -sS https://api.pulse.noelsabia.dev/api/readyz
```
Pass: the health loop shows at most a few seconds of errors, the deployment
returns to 1/1, `/api/stream` reconnects and emits a `scenario` event, and the
UI recovers without a manual reload. Note the actual outage duration in the
paper's reliability section.
Repeat for `web` (public page 502 briefly, then reloads) and `ml` (API serves
cached forecasts, no user-visible failure).
## Cleanup / invariants
- All deployments back to 1 replica.
- `make smoke` passes.
- No Ingress, Jobs, CronJobs, NetworkPolicies, or cluster-scoped resources were
created during the drills.
- Never leave `ml` or `cloudflared` scaled to zero.
+91
View File
@@ -0,0 +1,91 @@
# k6 load test
`k6/sse.js` holds 120 concurrent Server-Sent Events connections for 5 minutes
and reports connection count plus p95 event latency.
## Why a custom k6 binary
Native k6 buffers a complete HTTP response, so it cannot parse a stream that
never ends. The script imports the community `k6/x/sse` extension
(`github.com/phymbert/xk6-sse`). Newer k6 releases resolve community extensions
automatically. If the stock `grafana/k6` image rejects the import, build a
binary with the extension:
```bash
make k6-build # creates k6/k6-sse (git-ignored)
K6=./k6/k6-sse make load
```
`make load` uses, in order: `k6` on `PATH`, `k6/k6-sse`, then
`docker run --rm -i grafana/k6`.
## Procedure
1. Confirm the API is healthy and the SSE route works:
```bash
curl -sS https://api.pulse.noelsabia.dev/api/healthz
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream
```
You should see `retry: 3000` and a `scenario` event immediately, then a
`ping` every 15 s.
2. Run the load test:
```bash
make load
# or: API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js
```
`make load` writes `k6/results/summary-<timestamp>.json` (git-ignored).
3. During the run, post a scenario change so the scenario-event latency has
samples and the propagation path is exercised:
```bash
curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \
-H 'Content-Type: application/json' \
-d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \
https://api.pulse.noelsabia.dev/api/scenario
```
Repeat every 30–60 s with different scenarios if you want a distribution.
Without a change, `sse_event_latency` is measured from heartbeat `ping`
events, which still proves the stream is live.
4. Record these numbers in the paper/README from the saved summary:
- `sse_connections_opened` count (must be 120)
- `sse_connections_active` max (must reach 120)
- `sse_open_success` rate (must be >= 99%)
- `sse_event_latency` p95 (target < 2000 ms)
- `sse_connection_errors` (target 0)
- `sse_events_received` total
## Pass criteria
| Metric | Target |
|---|---|
| Connections opened | 120 |
| Peak active connections | 120 |
| Open success rate | >= 99% |
| p95 event latency | < 2000 ms |
| Unexpected connection errors | 0 |
| Duration | 5 minutes |
The k6 thresholds enforce the first, third, and fourth; the others are read
from the summary. A p95 above 2 s means Cloudflare, the Go SSE hub, or the
client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s
heartbeat, and whether the connection is HTTP/2-buffered at the edge.
## Notes
- `sse_event_latency` uses the server timestamp in `data.t` (ping) or `data.at`
(scenario). It includes clock skew between the k6 host and the pod; keep both
on NTP. Negative or > 1 h values are ignored.
- Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends
cleanly instead of waiting for the 7-minute request timeout.
- One API replica means all clients share one SSE hub; a pod restart drops
every stream and clients reconnect after `retry: 3000`.
- k6 itself does not collect server CPU/memory. Use `kubectl top pods` (if
metrics-server is available) or watch the pod during the run.
+172
View File
@@ -0,0 +1,172 @@
// PulseCast SSE load test: 120 concurrent clients for 5 minutes.
//
// The native k6 http module buffers the whole response, so it cannot read
// Server-Sent Events incrementally. This script uses the community
// `k6/x/sse` extension (github.com/phymbert/xk6-sse). Recent k6 releases
// resolve `k6/x/sse` automatically; otherwise build a binary with:
// make k6-build
//
// Run:
// API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js
// make load
//
// Recorded metrics:
// sse_connections_opened counter, expected >= SSE_VUS (120)
// sse_connections_active gauge, peak should reach 120
// sse_open_success rate of VUs that opened a stream
// sse_event_latency p95 server-timestamp-to-client latency, ms
// sse_scenario_latency same, scenario events only (if any occur)
// sse_ping_latency same, heartbeat events only
// sse_events_received total events
//
// Each VU closes its stream after SSE_HOLD_PINGS heartbeats (20 * 15 s = 5 min).
// That keeps 120 streams open for the full window and closes cleanly.
import http from 'k6/http';
import sse from 'k6/x/sse';
import { check } from 'k6';
import { Counter, Gauge, Rate, Trend } from 'k6/metrics';
const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, '');
const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`;
const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120));
const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20));
const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m';
const connectionsOpened = new Counter('sse_connections_opened');
const connectionsActive = new Gauge('sse_connections_active');
const openSuccess = new Rate('sse_open_success');
const connectionErrors = new Counter('sse_connection_errors');
const eventsReceived = new Counter('sse_events_received');
const eventLatency = new Trend('sse_event_latency', true);
const scenarioLatency = new Trend('sse_scenario_latency', true);
const pingLatency = new Trend('sse_ping_latency', true);
const thresholds = {
sse_open_success: ['rate>=0.99'],
sse_event_latency: ['p(95)<2000'],
};
thresholds[`sse_connections_opened`] = [`count>=${VUS}`];
export const options = {
scenarios: {
sse_clients: {
executor: 'per-vu-iterations',
vus: VUS,
iterations: 1,
maxDuration: MAX_DURATION,
gracefulStop: '30s',
},
},
thresholds,
// Every VU owns one long-lived connection; do not pool them.
noConnectionReuse: true,
summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max', 'count'],
};
export function setup() {
const res = http.get(`${API_URL}/api/healthz`, { tags: { endpoint: 'setup' } });
if (res.status !== 200) {
throw new Error(`API healthz returned ${res.status}; refusing to start the SSE load test`);
}
return { api: API_URL, stream: STREAM_URL, vus: VUS };
}
export default function () {
let opened = false;
let active = false;
let closedByUs = false;
let pings = 0;
const markActive = () => {
if (!active) {
active = true;
connectionsActive.add(1);
}
};
const markInactive = () => {
if (active) {
active = false;
connectionsActive.add(-1);
}
};
const params = {
method: 'GET',
headers: {
Accept: 'text/event-stream',
'Cache-Control': 'no-cache',
},
// Outlive the scenario; the 20th heartbeat closes the stream first.
timeout: __ENV.SSE_TIMEOUT || '7m',
tags: { endpoint: 'sse' },
};
const res = sse.open(STREAM_URL, params, function (client) {
client.on('open', function () {
opened = true;
markActive();
connectionsOpened.add(1);
});
client.on('event', function (event) {
eventsReceived.add(1);
const name = event.name || 'message';
const now = Date.now();
let payload = null;
if (event.data) {
try {
payload = JSON.parse(event.data);
} catch (_) {
payload = null;
}
}
// The contract puts the emit time in `t` (ping) or `at` (scenario).
// Latency is server timestamp -> client receipt, in milliseconds.
const stamp = payload && (payload.t || payload.at);
if (stamp) {
const emitted = Date.parse(stamp);
if (!Number.isNaN(emitted)) {
const lag = now - emitted;
// Ignore bad stamps and absurd clock skew (> 1 h).
if (lag >= 0 && lag < 3600000) {
eventLatency.add(lag);
if (name === 'scenario') scenarioLatency.add(lag);
if (name === 'ping') pingLatency.add(lag);
}
}
}
if (name === 'ping') {
pings += 1;
if (pings >= HOLD_PINGS && !closedByUs) {
closedByUs = true;
markInactive();
client.close();
}
}
});
client.on('error', function (e) {
if (!closedByUs) {
connectionErrors.add(1);
console.error(`SSE error: ${e && e.error ? e.error() : e}`);
}
markInactive();
});
});
check(res, {
'sse status 200': (r) => r && r.status === 200,
'sse content-type': (r) =>
r && String(r.headers['Content-Type'] || '').includes('text/event-stream'),
'sse opened': () => opened,
});
if (!opened) {
connectionErrors.add(1);
}
openSuccess.add(opened);
markInactive();
}
+4
View File
@@ -0,0 +1,4 @@
{
"$schema": "https://opencode.ai/config.json",
"model": "opencode-go/deepseek-v4.1-flash"
}
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env bash
# PulseCast smoke test.
#
# Defaults target the public TLS endpoints. Override for local dev:
# WEB_URL=http://localhost:3000 API_URL=http://localhost:8080 \
# ML_URL=http://localhost:8000 ./scripts/smoke.sh
#
# FastAPI is not public. If ML_URL is empty, the script uses a short
# `kubectl port-forward svc/ml` (requires KUBECONFIG). Set SKIP_ML=1 to skip.
# Set SMOKE_SSE=1 to also check that /api/stream emits an event.
set -uo pipefail
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}"
ML_URL="${ML_URL:-}"
NS="${NS:-fch26-onemanarmy}"
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"
CURL_MAX_TIME="${CURL_MAX_TIME:-20}"
PORT_FORWARD_PORT="${PORT_FORWARD_PORT:-18000}"
SMOKE_SSE="${SMOKE_SSE:-0}"
SKIP_ML="${SKIP_ML:-0}"
passes=0
failures=0
tmpdir="$(mktemp -d)"
pf_pid=""
cleanup() {
if [[ -n "$pf_pid" ]] && kill -0 "$pf_pid" 2>/dev/null; then
kill "$pf_pid" 2>/dev/null || true
wait "$pf_pid" 2>/dev/null || true
fi
rm -rf "$tmpdir"
}
trap cleanup EXIT INT TERM
pass() { printf 'PASS %s\n' "$*"; passes=$((passes + 1)); }
fail() { printf 'FAIL %s\n' "$*"; failures=$((failures + 1)); }
warn() { printf 'WARN %s\n' "$*"; }
check_http() {
local name="$1" url="$2" expect="${3:-200}" body_re="${4:-}"
local body="$tmpdir/body" code
code="$(curl -sS -L --max-time "$CURL_MAX_TIME" -o "$body" -w '%{http_code}' "$url" 2>"$tmpdir/curl.err")" || code="000"
if [[ "$code" != "$expect" ]]; then
fail "$name -> HTTP $code (expected $expect): $url"
[[ -s "$tmpdir/curl.err" ]] && sed 's/^/ /' "$tmpdir/curl.err"
return 1
fi
if [[ -n "$body_re" ]] && ! grep -Eq "$body_re" "$body"; then
fail "$name -> HTTP $code but body did not match /$body_re/"
head -c 300 "$body" | sed 's/^/ /'
[[ -s "$body" ]] && echo
return 1
fi
pass "$name -> HTTP $code"
return 0
}
check_https() {
local name="$1" url="$2" verify
case "$url" in
https://*) ;;
*) warn "$name is not HTTPS ($url); TLS check skipped"; return 0 ;;
esac
verify="$(curl -sS -o /dev/null --max-time "$CURL_MAX_TIME" -w '%{ssl_verify_result}' "$url" 2>/dev/null)" || verify="error"
if [[ "$verify" == "0" ]]; then
pass "$name -> TLS verified"
else
fail "$name -> TLS verification failed (ssl_verify_result=$verify)"
fi
}
resolve_ml_url() {
[[ -n "$ML_URL" ]] && return 0
command -v kubectl >/dev/null 2>&1 || return 1
local kargs=(kubectl)
[[ -n "$CONTEXT" ]] && kargs+=(--context "$CONTEXT")
echo " starting port-forward svc/ml -> 127.0.0.1:${PORT_FORWARD_PORT}" >&2
"${kargs[@]}" -n "$NS" port-forward svc/ml "${PORT_FORWARD_PORT}:8000" >"$tmpdir/pf.log" 2>&1 &
pf_pid=$!
local i
for i in $(seq 1 40); do
if curl -sS --max-time 2 -o /dev/null "http://127.0.0.1:${PORT_FORWARD_PORT}/healthz" 2>/dev/null; then
ML_URL="http://127.0.0.1:${PORT_FORWARD_PORT}"
return 0
fi
kill -0 "$pf_pid" 2>/dev/null || break
sleep 0.5
done
[[ -s "$tmpdir/pf.log" ]] && sed 's/^/ /' "$tmpdir/pf.log" >&2
return 1
}
check_sse() {
local out="$tmpdir/sse"
# The stream never ends by design; stop after one heartbeat interval + margin.
timeout 20 curl -sS -N --max-time 20 -H 'Accept: text/event-stream' \
"$API_URL/api/stream" >"$out" 2>"$tmpdir/sse.err" || true
if grep -Eq '^(event: (scenario|ping)|data: )' "$out"; then
pass "SSE /api/stream -> event received"
else
fail "SSE /api/stream -> no event within 20s"
head -c 300 "$out" | sed 's/^/ /'
[[ -s "$tmpdir/sse.err" ]] && sed 's/^/ /' "$tmpdir/sse.err"
fi
}
printf 'PulseCast smoke\n'
printf ' web: %s\n api: %s\n ml: %s\n\n' "$WEB_URL" "$API_URL" "${ML_URL:-<port-forward svc/ml>}"
check_http "web /" "$WEB_URL/" 200
check_https "web /" "$WEB_URL"
check_http "api /api/healthz" "$API_URL/api/healthz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
check_http "api /api/readyz" "$API_URL/api/readyz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
check_https "api /api/healthz" "$API_URL/api/healthz"
if [[ "$SKIP_ML" == "1" ]]; then
warn "ml /healthz skipped (SKIP_ML=1)"
elif resolve_ml_url; then
check_http "ml /healthz" "$ML_URL/healthz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
else
fail "ml /healthz -> cannot reach FastAPI (set ML_URL, export KUBECONFIG, or SKIP_ML=1)"
fi
if [[ "$SMOKE_SSE" == "1" ]]; then
check_sse
fi
printf '\n%d passed, %d failed\n' "$passes" "$failures"
[[ "$failures" -eq 0 ]]