From aeee20cb7f20caf2903523ed2f3c70b730c73aba Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 04:16:22 +0100 Subject: [PATCH] chore: bootstrap infra-k8 scaffold --- .gitignore | 17 +++ Makefile | 153 ++++++++++++++++++++++++ README.md | 48 ++++++++ deploy/examples/secrets.example.yaml | 43 +++++++ deploy/k8s/00-config.yaml | 63 ++++++++++ deploy/k8s/10-ml.yaml | 107 +++++++++++++++++ deploy/k8s/20-api.yaml | 118 ++++++++++++++++++ deploy/k8s/30-web.yaml | 111 +++++++++++++++++ deploy/k8s/40-cloudflared.yaml | 99 +++++++++++++++ docs/failure-drills.md | 96 +++++++++++++++ docs/load-test.md | 91 ++++++++++++++ k6/sse.js | 172 +++++++++++++++++++++++++++ opencode.json | 4 + scripts/smoke.sh | 134 +++++++++++++++++++++ 14 files changed, 1256 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 deploy/examples/secrets.example.yaml create mode 100644 deploy/k8s/00-config.yaml create mode 100644 deploy/k8s/10-ml.yaml create mode 100644 deploy/k8s/20-api.yaml create mode 100644 deploy/k8s/30-web.yaml create mode 100644 deploy/k8s/40-cloudflared.yaml create mode 100644 docs/failure-drills.md create mode 100644 docs/load-test.md create mode 100644 k6/sse.js create mode 100644 opencode.json create mode 100644 scripts/smoke.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6de1550 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# Local secrets and cluster credentials +*kubeconfig* +*.kubeconfig +.env +.env.* +secrets.local.yaml +deploy/secrets*.yaml + +# k6 binaries and run output +k6/k6-sse +k6/xk6 +k6/results/* +!k6/results/.gitkeep + +# Editor / OS noise +.DS_Store +*.swp diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..04a4f46 --- /dev/null +++ b/Makefile @@ -0,0 +1,153 @@ +SHELL := /usr/bin/env bash +.DEFAULT_GOAL := help + +ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) + +NAMESPACE ?= fch26-onemanarmy +CONTEXT ?= fch26-onemanarmy-sa-context +KUBECTL ?= kubectl --context $(CONTEXT) --namespace $(NAMESPACE) + +REGISTRY ?= git.noelsabia.dev +OWNER ?= noelsabia +TAG ?= gate1 +WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG) +API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG) +ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG) + +WEB_DIR ?= $(ROOT)/../web-nuxt +API_DIR ?= $(ROOT)/../api-go +ML_DIR ?= $(ROOT)/../ML-py + +WEB_URL ?= https://pulse.noelsabia.dev +API_URL ?= https://api.pulse.noelsabia.dev +LOAD_API_URL ?= $(API_URL) +SSE_VUS ?= 120 +SSE_HOLD_PINGS ?= 20 +SSE_MAX_DURATION ?= 6m +K6_IMAGE ?= grafana/k6:latest +K6_SCRIPT := $(ROOT)/k6/sse.js +K6_RESULTS := $(ROOT)/k6/results + +.PHONY: help validate validate-kubeconform build build-web build-api build-ml push push-web push-api push-ml set-image deploy deploy-check status logs smoke smoke-local load k6-build secret-commands clean + +help: ## Show available targets + @awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +validate: ## Client-side dry-run of deploy/k8s (never applies) + @command -v kubectl >/dev/null 2>&1 || { echo "kubectl not found"; exit 1; } + kubectl apply --dry-run=client --validate=false -f $(ROOT)/deploy/k8s + @if command -v kubeconform >/dev/null 2>&1; then \ + kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s; \ + else \ + echo "kubeconform not installed; kubectl client dry-run is the schema check"; \ + fi + +validate-kubeconform: ## Validate schemas with kubeconform if installed + kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s + +build: build-web build-api build-ml ## Build all three images locally + +build-web: ## Build the web image + @test -f $(WEB_DIR)/Dockerfile || { echo "missing $(WEB_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; } + docker build -t $(WEB_IMAGE) $(WEB_DIR) + +build-api: ## Build the api image + @test -f $(API_DIR)/Dockerfile || { echo "missing $(API_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; } + docker build -t $(API_IMAGE) $(API_DIR) + +build-ml: ## Build the ml image + @test -f $(ML_DIR)/Dockerfile || { echo "missing $(ML_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; } + docker build -t $(ML_IMAGE) $(ML_DIR) + +push: push-web push-api push-ml ## Push all three images to Gitea + +push-web: ## Push the web image + docker push $(WEB_IMAGE) + +push-api: ## Push the api image + docker push $(API_IMAGE) + +push-ml: ## Push the ml image + docker push $(ML_IMAGE) + +set-image: ## Update live Deployments to TAG (guarded; human only) + @test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; } + $(KUBECTL) set image deployment/web web=$(WEB_IMAGE) + $(KUBECTL) set image deployment/api api=$(API_IMAGE) + $(KUBECTL) set image deployment/ml ml=$(ML_IMAGE) + $(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s + +deploy-check: ## Verify required Secrets exist (read-only) + @for s in gitea-regcred cloudflared control-auth; do \ + $(KUBECTL) get secret "$$s" >/dev/null 2>&1 || { \ + echo "Missing Secret $$s in $(NAMESPACE); see docs/deploy.md"; exit 1; }; \ + done + @echo "required Secrets present" + +deploy: deploy-check ## Apply manifests (guarded; human only) + @test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; } + $(KUBECTL) apply -f $(ROOT)/deploy/k8s + $(KUBECTL) rollout status deployment/ml --timeout=180s + $(KUBECTL) rollout status deployment/api --timeout=180s + $(KUBECTL) rollout status deployment/web --timeout=180s + $(KUBECTL) rollout status deployment/cloudflared --timeout=120s + @echo + @echo "Deployed. Now run: make smoke" + +status: ## Show deployments, pods, services, and secrets + $(KUBECTL) get deploy,pods,svc,secrets + +logs: ## Follow cloudflared logs + $(KUBECTL) logs -f deployment/cloudflared + +smoke: ## Public HTTPS + API + ML health checks + WEB_URL="$(WEB_URL)" API_URL="$(API_URL)" NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" \ + $(ROOT)/scripts/smoke.sh + +smoke-local: ## Smoke the three local dev servers + WEB_URL="http://localhost:3000" API_URL="http://localhost:8080" \ + ML_URL="http://localhost:8000" $(ROOT)/scripts/smoke.sh + +load: ## Run the 120-client / 5-minute SSE load test + @mkdir -p $(K6_RESULTS) + @export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" \ + SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \ + stamp=$$(date +%Y%m%dT%H%M%S); \ + if command -v k6 >/dev/null 2>&1; then \ + echo "Using k6 from PATH"; \ + k6 run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \ + elif [ -x "$(ROOT)/k6/k6-sse" ]; then \ + echo "Using $(ROOT)/k6/k6-sse"; \ + "$(ROOT)/k6/k6-sse" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \ + else \ + echo "Using docker image $(K6_IMAGE)"; \ + docker run --rm -i \ + -v "$(ROOT)/k6:/scripts:ro" \ + -v "$(K6_RESULTS):/results" \ + -e API_URL -e SSE_VUS -e SSE_HOLD_PINGS -e SSE_MAX_DURATION \ + $(K6_IMAGE) run --summary-export="/results/summary-$$stamp.json" /scripts/sse.js; \ + fi + +k6-build: ## Build a k6 binary with the SSE extension (k6/k6-sse) + @mkdir -p $(K6_RESULTS) + cd $(ROOT)/k6 && GOBIN=$(ROOT)/k6 go install go.k6.io/xk6/cmd/xk6@latest + cd $(ROOT)/k6 && $(ROOT)/k6/xk6 build --with github.com/phymbert/xk6-sse@latest --output $(ROOT)/k6/k6-sse + @echo "Built $(ROOT)/k6/k6-sse" + +secret-commands: ## Print the exact Secret creation commands (creates nothing) + @echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic cloudflared \\" + @echo " --from-literal=TUNNEL_TOKEN=''" + @echo + @echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic control-auth \\" + @echo " --from-literal=CONTROL_USER='' \\" + @echo " --from-literal=CONTROL_PASS=''" + @echo + @echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret docker-registry gitea-regcred \\" + @echo " --docker-server=$(REGISTRY) \\" + @echo " --docker-username='' \\" + @echo " --docker-password=''" + +clean: ## Remove local k6 build artifacts and results + rm -f $(ROOT)/k6/k6-sse $(ROOT)/k6/xk6 + rm -rf $(K6_RESULTS) + @mkdir -p $(K6_RESULTS) && touch $(K6_RESULTS)/.gitkeep diff --git a/README.md b/README.md new file mode 100644 index 0000000..d56f90c --- /dev/null +++ b/README.md @@ -0,0 +1,48 @@ +# PulseCast infra/QA + +Namespace-only Kubernetes manifests, smoke tests, and the k6 SSE load test for +PulseCast. This repo never applies cluster changes by itself: the human runs the +commands in `docs/deploy.md`. + +## Layout + +```text +deploy/k8s/ web, api, ml + cloudflared Deployments and Services, ConfigMaps +deploy/examples/ example Secret shapes (placeholders only, never applied) +scripts/smoke.sh public + in-cluster smoke checks +k6/sse.js 120 concurrent SSE clients for 5 minutes +docs/deploy.md Gate 1 runbook and exact human checklist +docs/load-test.md load-test procedure and how to record results +docs/failure-drills.md FastAPI-down, tunnel-down, pod-restart drills +Makefile validate, build, push, deploy, smoke, load helpers +``` + +## Quick start + +```bash +export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml # chmod 600 +make validate # client-side dry run only, never applies +make deploy CONFIRM=yes # human only; requires the three Secrets first +make smoke # public HTTPS + API + ML health checks +make load # k6 SSE load test (needs the k6 SSE extension) +``` + +Read `docs/deploy.md` before the first deploy. It contains the Gitea token, +`docker login`, hello-world push, `gitea-regcred`, Cloudflare Tunnel, public +hostname, and verification steps. + +## Git workflow + +`main` is integration. New work happens on a `feat/` branch in its own +worktree. Before every push: `git fetch origin && git merge origin/main`. +Never force-push `main`. The initial scaffold is the one bootstrap exception +committed directly to `main`. + +## Safety rules + +- No Ingress, cert-manager, external-dns, Jobs, CronJobs, CRDs, Roles, + RoleBindings, NetworkPolicies, or any cluster-scoped resource. +- Exposure only through the `cloudflared` Deployment. +- No secrets, tokens, kubeconfig, or raw CSV in git. +- The human applies cluster mutations. `make deploy` refuses to run without + `CONFIRM=yes`. diff --git a/deploy/examples/secrets.example.yaml b/deploy/examples/secrets.example.yaml new file mode 100644 index 0000000..e7d0a0f --- /dev/null +++ b/deploy/examples/secrets.example.yaml @@ -0,0 +1,43 @@ +# EXAMPLE ONLY — do not apply this file. +# It exists so the manifest shapes are reviewable. Create the real secrets with +# the commands in docs/deploy.md; never commit real tokens or credentials. +# +# kubectl -n fch26-onemanarmy create secret generic cloudflared \ +# --from-literal=TUNNEL_TOKEN='' +# +# kubectl -n fch26-onemanarmy create secret generic control-auth \ +# --from-literal=CONTROL_USER='' \ +# --from-literal=CONTROL_PASS='' +# +# kubectl -n fch26-onemanarmy create secret docker-registry gitea-regcred \ +# --docker-server=git.noelsabia.dev \ +# --docker-username='' \ +# --docker-password='' +apiVersion: v1 +kind: Secret +metadata: + name: cloudflared + namespace: fch26-onemanarmy +type: Opaque +stringData: + TUNNEL_TOKEN: "REPLACE_ME" +--- +apiVersion: v1 +kind: Secret +metadata: + name: control-auth + namespace: fch26-onemanarmy +type: Opaque +stringData: + CONTROL_USER: "REPLACE_ME" + CONTROL_PASS: "REPLACE_ME" +--- +apiVersion: v1 +kind: Secret +metadata: + name: gitea-regcred + namespace: fch26-onemanarmy +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + {"auths":{"git.noelsabia.dev":{"username":"REPLACE_ME","password":"REPLACE_ME","auth":"REPLACE_ME"}}} diff --git a/deploy/k8s/00-config.yaml b/deploy/k8s/00-config.yaml new file mode 100644 index 0000000..a6c2ffc --- /dev/null +++ b/deploy/k8s/00-config.yaml @@ -0,0 +1,63 @@ +# Non-secret configuration for PulseCast. +# Namespace-scoped only. Secrets are created by the human, never committed. +apiVersion: v1 +kind: ConfigMap +metadata: + name: pulsecast-shared + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/part-of: pulsecast +data: + PUBLIC_WEB_URL: "https://pulse.noelsabia.dev" + PUBLIC_API_URL: "https://api.pulse.noelsabia.dev" + API_INTERNAL_URL: "http://api:8080" + ML_INTERNAL_URL: "http://ml:8000" + CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: web-config + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: pulsecast +data: + NODE_ENV: "production" + HOST: "0.0.0.0" + PORT: "3000" + NUXT_TELEMETRY_DISABLED: "1" + NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev" + NUXT_API_INTERNAL_BASE: "http://api:8080" + NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: api-config + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: api + app.kubernetes.io/part-of: pulsecast +data: + PORT: "8080" + DATA_DIR: "/app/data" + ML_BASE_URL: "http://ml:8000" + ML_TIMEOUT: "1.5s" + SSE_HEARTBEAT: "15s" + CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-config + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: ml + app.kubernetes.io/part-of: pulsecast +data: + PORT: "8000" + DATA_DIR: "/app/data" + MODEL_PATH: "/app/model/model.joblib" + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" diff --git a/deploy/k8s/10-ml.yaml b/deploy/k8s/10-ml.yaml new file mode 100644 index 0000000..39ccaeb --- /dev/null +++ b/deploy/k8s/10-ml.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: ml + app.kubernetes.io/component: ml + app.kubernetes.io/part-of: pulsecast +spec: + replicas: 1 + revisionHistoryLimit: 2 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: ml + template: + metadata: + labels: + app.kubernetes.io/name: ml + app.kubernetes.io/component: ml + app.kubernetes.io/part-of: pulsecast + spec: + automountServiceAccountToken: false + imagePullSecrets: + - name: gitea-regcred + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: ml + image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8000 + protocol: TCP + envFrom: + - configMapRef: + name: ml-config + - configMapRef: + name: pulsecast-shared + env: + - name: HOME + value: /tmp + - name: MPLCONFIGDIR + value: /tmp + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp + mountPath: /tmp + startupProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 2 + failureThreshold: 60 + livenessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /readyz + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + terminationGracePeriodSeconds: 30 + volumes: + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: ml + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: ml + app.kubernetes.io/part-of: pulsecast +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: ml + ports: + - name: http + port: 8000 + targetPort: http + protocol: TCP diff --git a/deploy/k8s/20-api.yaml b/deploy/k8s/20-api.yaml new file mode 100644 index 0000000..aeb4db9 --- /dev/null +++ b/deploy/k8s/20-api.yaml @@ -0,0 +1,118 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: api + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: pulsecast +spec: + # Scenario state and the SSE hub live in memory. One replica is intentional. + replicas: 1 + revisionHistoryLimit: 2 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: api + template: + metadata: + labels: + app.kubernetes.io/name: api + app.kubernetes.io/component: backend + app.kubernetes.io/part-of: pulsecast + spec: + automountServiceAccountToken: false + imagePullSecrets: + - name: gitea-regcred + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: api + image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 + protocol: TCP + envFrom: + - configMapRef: + name: api-config + - configMapRef: + name: pulsecast-shared + env: + - name: HOME + value: /tmp + # Created by the human; see docs/deploy.md. Never commit real credentials. + - name: CONTROL_USER + valueFrom: + secretKeyRef: + name: control-auth + key: CONTROL_USER + - name: CONTROL_PASS + valueFrom: + secretKeyRef: + name: control-auth + key: CONTROL_PASS + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp + mountPath: /tmp + startupProbe: + httpGet: + path: /api/healthz + port: http + periodSeconds: 2 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /api/healthz + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /api/readyz + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + # SSE clients reconnect with retry: 3000; give them a moment on shutdown. + terminationGracePeriodSeconds: 20 + volumes: + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: api + app.kubernetes.io/part-of: pulsecast +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: api + ports: + - name: http + port: 8080 + targetPort: http + protocol: TCP diff --git a/deploy/k8s/30-web.yaml b/deploy/k8s/30-web.yaml new file mode 100644 index 0000000..41a7557 --- /dev/null +++ b/deploy/k8s/30-web.yaml @@ -0,0 +1,111 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: web + app.kubernetes.io/component: frontend + app.kubernetes.io/part-of: pulsecast +spec: + replicas: 1 + revisionHistoryLimit: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: web + template: + metadata: + labels: + app.kubernetes.io/name: web + app.kubernetes.io/component: frontend + app.kubernetes.io/part-of: pulsecast + spec: + automountServiceAccountToken: false + imagePullSecrets: + - name: gitea-regcred + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: web + # Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`. + image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 3000 + protocol: TCP + envFrom: + - configMapRef: + name: web-config + - configMapRef: + name: pulsecast-shared + env: + - name: HOME + value: /tmp + # Nuxt/Nitro may use a temp dir; keep it on the writable emptyDir. + - name: NITRO_TEMP_DIR + value: /tmp + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp + mountPath: /tmp + startupProbe: + httpGet: + path: / + port: http + periodSeconds: 2 + failureThreshold: 30 + livenessProbe: + httpGet: + path: / + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: / + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + volumes: + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: web + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: pulsecast +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: web + ports: + - name: http + port: 3000 + targetPort: http + protocol: TCP diff --git a/deploy/k8s/40-cloudflared.yaml b/deploy/k8s/40-cloudflared.yaml new file mode 100644 index 0000000..1672a48 --- /dev/null +++ b/deploy/k8s/40-cloudflared.yaml @@ -0,0 +1,99 @@ +# Cloudflare Tunnel is the only public exposure path. +# Routes are configured in the Cloudflare Zero Trust dashboard: +# pulse.noelsabia.dev -> http://web:3000 +# api.pulse.noelsabia.dev -> http://api:8080 +# The token lives in the `cloudflared` Secret, created by the human; never in git. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloudflared + namespace: fch26-onemanarmy + labels: + app.kubernetes.io/name: cloudflared + app.kubernetes.io/component: tunnel + app.kubernetes.io/part-of: pulsecast +spec: + replicas: 1 + revisionHistoryLimit: 2 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: cloudflared + template: + metadata: + labels: + app.kubernetes.io/name: cloudflared + app.kubernetes.io/component: tunnel + app.kubernetes.io/part-of: pulsecast + spec: + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: cloudflared + # Public Docker Hub image; no Gitea pull secret needed for this pod. + image: cloudflare/cloudflared:latest + imagePullPolicy: IfNotPresent + args: + - tunnel + - --no-autoupdate + - --metrics + - 0.0.0.0:2000 + - run + - --token + - $(TUNNEL_TOKEN) + env: + - name: TUNNEL_TOKEN + valueFrom: + secretKeyRef: + name: cloudflared + key: TUNNEL_TOKEN + - name: HOME + value: /tmp + ports: + - name: metrics + containerPort: 2000 + protocol: TCP + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: tmp + mountPath: /tmp + startupProbe: + httpGet: + path: /ready + port: metrics + periodSeconds: 2 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /ready + port: metrics + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: metrics + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 2 + terminationGracePeriodSeconds: 30 + volumes: + - name: tmp + emptyDir: {} diff --git a/docs/failure-drills.md b/docs/failure-drills.md new file mode 100644 index 0000000..321e98b --- /dev/null +++ b/docs/failure-drills.md @@ -0,0 +1,96 @@ +# Failure drills + +Run these **after** Gate 1 and outside the demo window. Every command is a +cluster mutation: the human runs it, not an agent. Record the result and the +recovery time. Keep `make smoke` and a `curl -N .../api/stream` open while you +drill. + +## 1. FastAPI down (cached fallback) + +Goal: the Go API keeps serving forecasts from baked JSON with `"cached": true` +and `/api/readyz` stays 200. FastAPI being down must not fail API readiness. + +```bash +# Baseline +curl -sS https://api.pulse.noelsabia.dev/api/readyz +curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached' + +# Stop ML +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=0 + +# Verify fallback (allow up to 2 s for the ML timeout) +curl -sS https://api.pulse.noelsabia.dev/api/readyz +curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}' +# Expected: readyz 200; cached true; points still present + +# Recover +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=1 +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/ml --timeout=180s +``` + +Pass: no 5xx from `/api/forecast`, `cached` flips to `true` within the 1.5 s +timeout budget, and the UI shows the cached-forecast banner. Fail: a 502/503 or +an empty forecast. + +## 2. Tunnel down + +Goal: detect a public outage quickly, restart `cloudflared`, and verify both +public hostnames recover. Local `kubectl port-forward` is a diagnostic only, +never the public demo path. + +```bash +# Stop the tunnel +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/cloudflared --replicas=0 + +# Verify the outage from outside +curl -sS -o /dev/null -w '%{http_code}\n' https://pulse.noelsabia.dev +curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz +# Expected: 502/503/530-style edge error, not a valid 200 + +# Recover +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/cloudflared --replicas=1 +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/cloudflared --timeout=120s +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy logs deployment/cloudflared --tail=50 +# Look for: "Registered tunnel connection" + +make smoke +``` + +If the tunnel pod is Running but the URL is still 502, check the dashboard +routes (`pulse` -> `http://web:3000`, `api.pulse` -> `http://api:8080`) and the +Secret token. A bad token keeps the pod restarting with an auth error. Rotating +the tunnel token means updating the Secret and restarting the Deployment. + +## 3. Pod restart (API and SSE reconnect) + +Goal: a single API pod restart causes a short 502 window, not a stuck UI. SSE +clients reconnect with `retry: 3000`; scenario state is in memory and resets to +the default, which is documented. + +```bash +# Watch from two terminals: +# curl -sS -N https://api.pulse.noelsabia.dev/api/stream +# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz; sleep 1; done + +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy delete pod -l app.kubernetes.io/name=api + +kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/api --timeout=180s +curl -sS https://api.pulse.noelsabia.dev/api/healthz +curl -sS https://api.pulse.noelsabia.dev/api/readyz +``` + +Pass: the health loop shows at most a few seconds of errors, the deployment +returns to 1/1, `/api/stream` reconnects and emits a `scenario` event, and the +UI recovers without a manual reload. Note the actual outage duration in the +paper's reliability section. + +Repeat for `web` (public page 502 briefly, then reloads) and `ml` (API serves +cached forecasts, no user-visible failure). + +## Cleanup / invariants + +- All deployments back to 1 replica. +- `make smoke` passes. +- No Ingress, Jobs, CronJobs, NetworkPolicies, or cluster-scoped resources were + created during the drills. +- Never leave `ml` or `cloudflared` scaled to zero. diff --git a/docs/load-test.md b/docs/load-test.md new file mode 100644 index 0000000..4133251 --- /dev/null +++ b/docs/load-test.md @@ -0,0 +1,91 @@ +# k6 load test + +`k6/sse.js` holds 120 concurrent Server-Sent Events connections for 5 minutes +and reports connection count plus p95 event latency. + +## Why a custom k6 binary + +Native k6 buffers a complete HTTP response, so it cannot parse a stream that +never ends. The script imports the community `k6/x/sse` extension +(`github.com/phymbert/xk6-sse`). Newer k6 releases resolve community extensions +automatically. If the stock `grafana/k6` image rejects the import, build a +binary with the extension: + +```bash +make k6-build # creates k6/k6-sse (git-ignored) +K6=./k6/k6-sse make load +``` + +`make load` uses, in order: `k6` on `PATH`, `k6/k6-sse`, then +`docker run --rm -i grafana/k6`. + +## Procedure + +1. Confirm the API is healthy and the SSE route works: + + ```bash + curl -sS https://api.pulse.noelsabia.dev/api/healthz + curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream + ``` + + You should see `retry: 3000` and a `scenario` event immediately, then a + `ping` every 15 s. + +2. Run the load test: + + ```bash + make load + # or: API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js + ``` + + `make load` writes `k6/results/summary-.json` (git-ignored). + +3. During the run, post a scenario change so the scenario-event latency has + samples and the propagation path is exercised: + + ```bash + curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \ + -H 'Content-Type: application/json' \ + -d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \ + https://api.pulse.noelsabia.dev/api/scenario + ``` + + Repeat every 30–60 s with different scenarios if you want a distribution. + Without a change, `sse_event_latency` is measured from heartbeat `ping` + events, which still proves the stream is live. + +4. Record these numbers in the paper/README from the saved summary: + - `sse_connections_opened` count (must be 120) + - `sse_connections_active` max (must reach 120) + - `sse_open_success` rate (must be >= 99%) + - `sse_event_latency` p95 (target < 2000 ms) + - `sse_connection_errors` (target 0) + - `sse_events_received` total + +## Pass criteria + +| Metric | Target | +|---|---| +| Connections opened | 120 | +| Peak active connections | 120 | +| Open success rate | >= 99% | +| p95 event latency | < 2000 ms | +| Unexpected connection errors | 0 | +| Duration | 5 minutes | + +The k6 thresholds enforce the first, third, and fourth; the others are read +from the summary. A p95 above 2 s means Cloudflare, the Go SSE hub, or the +client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s +heartbeat, and whether the connection is HTTP/2-buffered at the edge. + +## Notes + +- `sse_event_latency` uses the server timestamp in `data.t` (ping) or `data.at` + (scenario). It includes clock skew between the k6 host and the pod; keep both + on NTP. Negative or > 1 h values are ignored. +- Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends + cleanly instead of waiting for the 7-minute request timeout. +- One API replica means all clients share one SSE hub; a pod restart drops + every stream and clients reconnect after `retry: 3000`. +- k6 itself does not collect server CPU/memory. Use `kubectl top pods` (if + metrics-server is available) or watch the pod during the run. diff --git a/k6/sse.js b/k6/sse.js new file mode 100644 index 0000000..af0f09e --- /dev/null +++ b/k6/sse.js @@ -0,0 +1,172 @@ +// PulseCast SSE load test: 120 concurrent clients for 5 minutes. +// +// The native k6 http module buffers the whole response, so it cannot read +// Server-Sent Events incrementally. This script uses the community +// `k6/x/sse` extension (github.com/phymbert/xk6-sse). Recent k6 releases +// resolve `k6/x/sse` automatically; otherwise build a binary with: +// make k6-build +// +// Run: +// API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js +// make load +// +// Recorded metrics: +// sse_connections_opened counter, expected >= SSE_VUS (120) +// sse_connections_active gauge, peak should reach 120 +// sse_open_success rate of VUs that opened a stream +// sse_event_latency p95 server-timestamp-to-client latency, ms +// sse_scenario_latency same, scenario events only (if any occur) +// sse_ping_latency same, heartbeat events only +// sse_events_received total events +// +// Each VU closes its stream after SSE_HOLD_PINGS heartbeats (20 * 15 s = 5 min). +// That keeps 120 streams open for the full window and closes cleanly. +import http from 'k6/http'; +import sse from 'k6/x/sse'; +import { check } from 'k6'; +import { Counter, Gauge, Rate, Trend } from 'k6/metrics'; + +const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, ''); +const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`; +const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120)); +const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20)); +const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m'; + +const connectionsOpened = new Counter('sse_connections_opened'); +const connectionsActive = new Gauge('sse_connections_active'); +const openSuccess = new Rate('sse_open_success'); +const connectionErrors = new Counter('sse_connection_errors'); +const eventsReceived = new Counter('sse_events_received'); +const eventLatency = new Trend('sse_event_latency', true); +const scenarioLatency = new Trend('sse_scenario_latency', true); +const pingLatency = new Trend('sse_ping_latency', true); + +const thresholds = { + sse_open_success: ['rate>=0.99'], + sse_event_latency: ['p(95)<2000'], +}; +thresholds[`sse_connections_opened`] = [`count>=${VUS}`]; + +export const options = { + scenarios: { + sse_clients: { + executor: 'per-vu-iterations', + vus: VUS, + iterations: 1, + maxDuration: MAX_DURATION, + gracefulStop: '30s', + }, + }, + thresholds, + // Every VU owns one long-lived connection; do not pool them. + noConnectionReuse: true, + summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max', 'count'], +}; + +export function setup() { + const res = http.get(`${API_URL}/api/healthz`, { tags: { endpoint: 'setup' } }); + if (res.status !== 200) { + throw new Error(`API healthz returned ${res.status}; refusing to start the SSE load test`); + } + return { api: API_URL, stream: STREAM_URL, vus: VUS }; +} + +export default function () { + let opened = false; + let active = false; + let closedByUs = false; + let pings = 0; + + const markActive = () => { + if (!active) { + active = true; + connectionsActive.add(1); + } + }; + const markInactive = () => { + if (active) { + active = false; + connectionsActive.add(-1); + } + }; + + const params = { + method: 'GET', + headers: { + Accept: 'text/event-stream', + 'Cache-Control': 'no-cache', + }, + // Outlive the scenario; the 20th heartbeat closes the stream first. + timeout: __ENV.SSE_TIMEOUT || '7m', + tags: { endpoint: 'sse' }, + }; + + const res = sse.open(STREAM_URL, params, function (client) { + client.on('open', function () { + opened = true; + markActive(); + connectionsOpened.add(1); + }); + + client.on('event', function (event) { + eventsReceived.add(1); + const name = event.name || 'message'; + const now = Date.now(); + + let payload = null; + if (event.data) { + try { + payload = JSON.parse(event.data); + } catch (_) { + payload = null; + } + } + + // The contract puts the emit time in `t` (ping) or `at` (scenario). + // Latency is server timestamp -> client receipt, in milliseconds. + const stamp = payload && (payload.t || payload.at); + if (stamp) { + const emitted = Date.parse(stamp); + if (!Number.isNaN(emitted)) { + const lag = now - emitted; + // Ignore bad stamps and absurd clock skew (> 1 h). + if (lag >= 0 && lag < 3600000) { + eventLatency.add(lag); + if (name === 'scenario') scenarioLatency.add(lag); + if (name === 'ping') pingLatency.add(lag); + } + } + } + + if (name === 'ping') { + pings += 1; + if (pings >= HOLD_PINGS && !closedByUs) { + closedByUs = true; + markInactive(); + client.close(); + } + } + }); + + client.on('error', function (e) { + if (!closedByUs) { + connectionErrors.add(1); + console.error(`SSE error: ${e && e.error ? e.error() : e}`); + } + markInactive(); + }); + }); + + check(res, { + 'sse status 200': (r) => r && r.status === 200, + 'sse content-type': (r) => + r && String(r.headers['Content-Type'] || '').includes('text/event-stream'), + 'sse opened': () => opened, + }); + + if (!opened) { + connectionErrors.add(1); + } + openSuccess.add(opened); + markInactive(); +} diff --git a/opencode.json b/opencode.json new file mode 100644 index 0000000..8c19d09 --- /dev/null +++ b/opencode.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://opencode.ai/config.json", + "model": "opencode-go/deepseek-v4.1-flash" +} diff --git a/scripts/smoke.sh b/scripts/smoke.sh new file mode 100644 index 0000000..eaab5ff --- /dev/null +++ b/scripts/smoke.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# PulseCast smoke test. +# +# Defaults target the public TLS endpoints. Override for local dev: +# WEB_URL=http://localhost:3000 API_URL=http://localhost:8080 \ +# ML_URL=http://localhost:8000 ./scripts/smoke.sh +# +# FastAPI is not public. If ML_URL is empty, the script uses a short +# `kubectl port-forward svc/ml` (requires KUBECONFIG). Set SKIP_ML=1 to skip. +# Set SMOKE_SSE=1 to also check that /api/stream emits an event. +set -uo pipefail + +WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}" +API_URL="${API_URL:-https://api.pulse.noelsabia.dev}" +ML_URL="${ML_URL:-}" +NS="${NS:-fch26-onemanarmy}" +CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}" +CURL_MAX_TIME="${CURL_MAX_TIME:-20}" +PORT_FORWARD_PORT="${PORT_FORWARD_PORT:-18000}" +SMOKE_SSE="${SMOKE_SSE:-0}" +SKIP_ML="${SKIP_ML:-0}" + +passes=0 +failures=0 +tmpdir="$(mktemp -d)" +pf_pid="" + +cleanup() { + if [[ -n "$pf_pid" ]] && kill -0 "$pf_pid" 2>/dev/null; then + kill "$pf_pid" 2>/dev/null || true + wait "$pf_pid" 2>/dev/null || true + fi + rm -rf "$tmpdir" +} +trap cleanup EXIT INT TERM + +pass() { printf 'PASS %s\n' "$*"; passes=$((passes + 1)); } +fail() { printf 'FAIL %s\n' "$*"; failures=$((failures + 1)); } +warn() { printf 'WARN %s\n' "$*"; } + +check_http() { + local name="$1" url="$2" expect="${3:-200}" body_re="${4:-}" + local body="$tmpdir/body" code + code="$(curl -sS -L --max-time "$CURL_MAX_TIME" -o "$body" -w '%{http_code}' "$url" 2>"$tmpdir/curl.err")" || code="000" + if [[ "$code" != "$expect" ]]; then + fail "$name -> HTTP $code (expected $expect): $url" + [[ -s "$tmpdir/curl.err" ]] && sed 's/^/ /' "$tmpdir/curl.err" + return 1 + fi + if [[ -n "$body_re" ]] && ! grep -Eq "$body_re" "$body"; then + fail "$name -> HTTP $code but body did not match /$body_re/" + head -c 300 "$body" | sed 's/^/ /' + [[ -s "$body" ]] && echo + return 1 + fi + pass "$name -> HTTP $code" + return 0 +} + +check_https() { + local name="$1" url="$2" verify + case "$url" in + https://*) ;; + *) warn "$name is not HTTPS ($url); TLS check skipped"; return 0 ;; + esac + verify="$(curl -sS -o /dev/null --max-time "$CURL_MAX_TIME" -w '%{ssl_verify_result}' "$url" 2>/dev/null)" || verify="error" + if [[ "$verify" == "0" ]]; then + pass "$name -> TLS verified" + else + fail "$name -> TLS verification failed (ssl_verify_result=$verify)" + fi +} + +resolve_ml_url() { + [[ -n "$ML_URL" ]] && return 0 + command -v kubectl >/dev/null 2>&1 || return 1 + + local kargs=(kubectl) + [[ -n "$CONTEXT" ]] && kargs+=(--context "$CONTEXT") + echo " starting port-forward svc/ml -> 127.0.0.1:${PORT_FORWARD_PORT}" >&2 + "${kargs[@]}" -n "$NS" port-forward svc/ml "${PORT_FORWARD_PORT}:8000" >"$tmpdir/pf.log" 2>&1 & + pf_pid=$! + + local i + for i in $(seq 1 40); do + if curl -sS --max-time 2 -o /dev/null "http://127.0.0.1:${PORT_FORWARD_PORT}/healthz" 2>/dev/null; then + ML_URL="http://127.0.0.1:${PORT_FORWARD_PORT}" + return 0 + fi + kill -0 "$pf_pid" 2>/dev/null || break + sleep 0.5 + done + [[ -s "$tmpdir/pf.log" ]] && sed 's/^/ /' "$tmpdir/pf.log" >&2 + return 1 +} + +check_sse() { + local out="$tmpdir/sse" + # The stream never ends by design; stop after one heartbeat interval + margin. + timeout 20 curl -sS -N --max-time 20 -H 'Accept: text/event-stream' \ + "$API_URL/api/stream" >"$out" 2>"$tmpdir/sse.err" || true + if grep -Eq '^(event: (scenario|ping)|data: )' "$out"; then + pass "SSE /api/stream -> event received" + else + fail "SSE /api/stream -> no event within 20s" + head -c 300 "$out" | sed 's/^/ /' + [[ -s "$tmpdir/sse.err" ]] && sed 's/^/ /' "$tmpdir/sse.err" + fi +} + +printf 'PulseCast smoke\n' +printf ' web: %s\n api: %s\n ml: %s\n\n' "$WEB_URL" "$API_URL" "${ML_URL:-}" + +check_http "web /" "$WEB_URL/" 200 +check_https "web /" "$WEB_URL" + +check_http "api /api/healthz" "$API_URL/api/healthz" 200 '"ok"[[:space:]]*:[[:space:]]*true' +check_http "api /api/readyz" "$API_URL/api/readyz" 200 '"ok"[[:space:]]*:[[:space:]]*true' +check_https "api /api/healthz" "$API_URL/api/healthz" + +if [[ "$SKIP_ML" == "1" ]]; then + warn "ml /healthz skipped (SKIP_ML=1)" +elif resolve_ml_url; then + check_http "ml /healthz" "$ML_URL/healthz" 200 '"ok"[[:space:]]*:[[:space:]]*true' +else + fail "ml /healthz -> cannot reach FastAPI (set ML_URL, export KUBECONFIG, or SKIP_ML=1)" +fi + +if [[ "$SMOKE_SSE" == "1" ]]; then + check_sse +fi + +printf '\n%d passed, %d failed\n' "$passes" "$failures" +[[ "$failures" -eq 0 ]]