chore: bootstrap infra-k8 scaffold
This commit is contained in:
commit
aeee20cb7f
14 files changed
+1256
No files matched your search
+17
@@ -0,0 +1,17 @@
|
|||||||
|
# Local secrets and cluster credentials
|
||||||
|
*kubeconfig*
|
||||||
|
*.kubeconfig
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
secrets.local.yaml
|
||||||
|
deploy/secrets*.yaml
|
||||||
|
|
||||||
|
# k6 binaries and run output
|
||||||
|
k6/k6-sse
|
||||||
|
k6/xk6
|
||||||
|
k6/results/*
|
||||||
|
!k6/results/.gitkeep
|
||||||
|
|
||||||
|
# Editor / OS noise
|
||||||
|
.DS_Store
|
||||||
|
*.swp
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
SHELL := /usr/bin/env bash
|
||||||
|
.DEFAULT_GOAL := help
|
||||||
|
|
||||||
|
ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
|
||||||
|
|
||||||
|
NAMESPACE ?= fch26-onemanarmy
|
||||||
|
CONTEXT ?= fch26-onemanarmy-sa-context
|
||||||
|
KUBECTL ?= kubectl --context $(CONTEXT) --namespace $(NAMESPACE)
|
||||||
|
|
||||||
|
REGISTRY ?= git.noelsabia.dev
|
||||||
|
OWNER ?= noelsabia
|
||||||
|
TAG ?= gate1
|
||||||
|
WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
|
||||||
|
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
|
||||||
|
ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG)
|
||||||
|
|
||||||
|
WEB_DIR ?= $(ROOT)/../web-nuxt
|
||||||
|
API_DIR ?= $(ROOT)/../api-go
|
||||||
|
ML_DIR ?= $(ROOT)/../ML-py
|
||||||
|
|
||||||
|
WEB_URL ?= https://pulse.noelsabia.dev
|
||||||
|
API_URL ?= https://api.pulse.noelsabia.dev
|
||||||
|
LOAD_API_URL ?= $(API_URL)
|
||||||
|
SSE_VUS ?= 120
|
||||||
|
SSE_HOLD_PINGS ?= 20
|
||||||
|
SSE_MAX_DURATION ?= 6m
|
||||||
|
K6_IMAGE ?= grafana/k6:latest
|
||||||
|
K6_SCRIPT := $(ROOT)/k6/sse.js
|
||||||
|
K6_RESULTS := $(ROOT)/k6/results
|
||||||
|
|
||||||
|
.PHONY: help validate validate-kubeconform build build-web build-api build-ml push push-web push-api push-ml set-image deploy deploy-check status logs smoke smoke-local load k6-build secret-commands clean
|
||||||
|
|
||||||
|
help: ## Show available targets
|
||||||
|
@awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
||||||
|
|
||||||
|
validate: ## Client-side dry-run of deploy/k8s (never applies)
|
||||||
|
@command -v kubectl >/dev/null 2>&1 || { echo "kubectl not found"; exit 1; }
|
||||||
|
kubectl apply --dry-run=client --validate=false -f $(ROOT)/deploy/k8s
|
||||||
|
@if command -v kubeconform >/dev/null 2>&1; then \
|
||||||
|
kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s; \
|
||||||
|
else \
|
||||||
|
echo "kubeconform not installed; kubectl client dry-run is the schema check"; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
validate-kubeconform: ## Validate schemas with kubeconform if installed
|
||||||
|
kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s
|
||||||
|
|
||||||
|
build: build-web build-api build-ml ## Build all three images locally
|
||||||
|
|
||||||
|
build-web: ## Build the web image
|
||||||
|
@test -f $(WEB_DIR)/Dockerfile || { echo "missing $(WEB_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
|
||||||
|
docker build -t $(WEB_IMAGE) $(WEB_DIR)
|
||||||
|
|
||||||
|
build-api: ## Build the api image
|
||||||
|
@test -f $(API_DIR)/Dockerfile || { echo "missing $(API_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
|
||||||
|
docker build -t $(API_IMAGE) $(API_DIR)
|
||||||
|
|
||||||
|
build-ml: ## Build the ml image
|
||||||
|
@test -f $(ML_DIR)/Dockerfile || { echo "missing $(ML_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
|
||||||
|
docker build -t $(ML_IMAGE) $(ML_DIR)
|
||||||
|
|
||||||
|
push: push-web push-api push-ml ## Push all three images to Gitea
|
||||||
|
|
||||||
|
push-web: ## Push the web image
|
||||||
|
docker push $(WEB_IMAGE)
|
||||||
|
|
||||||
|
push-api: ## Push the api image
|
||||||
|
docker push $(API_IMAGE)
|
||||||
|
|
||||||
|
push-ml: ## Push the ml image
|
||||||
|
docker push $(ML_IMAGE)
|
||||||
|
|
||||||
|
set-image: ## Update live Deployments to TAG (guarded; human only)
|
||||||
|
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
|
||||||
|
$(KUBECTL) set image deployment/web web=$(WEB_IMAGE)
|
||||||
|
$(KUBECTL) set image deployment/api api=$(API_IMAGE)
|
||||||
|
$(KUBECTL) set image deployment/ml ml=$(ML_IMAGE)
|
||||||
|
$(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s
|
||||||
|
|
||||||
|
deploy-check: ## Verify required Secrets exist (read-only)
|
||||||
|
@for s in gitea-regcred cloudflared control-auth; do \
|
||||||
|
$(KUBECTL) get secret "$$s" >/dev/null 2>&1 || { \
|
||||||
|
echo "Missing Secret $$s in $(NAMESPACE); see docs/deploy.md"; exit 1; }; \
|
||||||
|
done
|
||||||
|
@echo "required Secrets present"
|
||||||
|
|
||||||
|
deploy: deploy-check ## Apply manifests (guarded; human only)
|
||||||
|
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
|
||||||
|
$(KUBECTL) apply -f $(ROOT)/deploy/k8s
|
||||||
|
$(KUBECTL) rollout status deployment/ml --timeout=180s
|
||||||
|
$(KUBECTL) rollout status deployment/api --timeout=180s
|
||||||
|
$(KUBECTL) rollout status deployment/web --timeout=180s
|
||||||
|
$(KUBECTL) rollout status deployment/cloudflared --timeout=120s
|
||||||
|
@echo
|
||||||
|
@echo "Deployed. Now run: make smoke"
|
||||||
|
|
||||||
|
status: ## Show deployments, pods, services, and secrets
|
||||||
|
$(KUBECTL) get deploy,pods,svc,secrets
|
||||||
|
|
||||||
|
logs: ## Follow cloudflared logs
|
||||||
|
$(KUBECTL) logs -f deployment/cloudflared
|
||||||
|
|
||||||
|
smoke: ## Public HTTPS + API + ML health checks
|
||||||
|
WEB_URL="$(WEB_URL)" API_URL="$(API_URL)" NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" \
|
||||||
|
$(ROOT)/scripts/smoke.sh
|
||||||
|
|
||||||
|
smoke-local: ## Smoke the three local dev servers
|
||||||
|
WEB_URL="http://localhost:3000" API_URL="http://localhost:8080" \
|
||||||
|
ML_URL="http://localhost:8000" $(ROOT)/scripts/smoke.sh
|
||||||
|
|
||||||
|
load: ## Run the 120-client / 5-minute SSE load test
|
||||||
|
@mkdir -p $(K6_RESULTS)
|
||||||
|
@export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" \
|
||||||
|
SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \
|
||||||
|
stamp=$$(date +%Y%m%dT%H%M%S); \
|
||||||
|
if command -v k6 >/dev/null 2>&1; then \
|
||||||
|
echo "Using k6 from PATH"; \
|
||||||
|
k6 run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \
|
||||||
|
elif [ -x "$(ROOT)/k6/k6-sse" ]; then \
|
||||||
|
echo "Using $(ROOT)/k6/k6-sse"; \
|
||||||
|
"$(ROOT)/k6/k6-sse" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \
|
||||||
|
else \
|
||||||
|
echo "Using docker image $(K6_IMAGE)"; \
|
||||||
|
docker run --rm -i \
|
||||||
|
-v "$(ROOT)/k6:/scripts:ro" \
|
||||||
|
-v "$(K6_RESULTS):/results" \
|
||||||
|
-e API_URL -e SSE_VUS -e SSE_HOLD_PINGS -e SSE_MAX_DURATION \
|
||||||
|
$(K6_IMAGE) run --summary-export="/results/summary-$$stamp.json" /scripts/sse.js; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
k6-build: ## Build a k6 binary with the SSE extension (k6/k6-sse)
|
||||||
|
@mkdir -p $(K6_RESULTS)
|
||||||
|
cd $(ROOT)/k6 && GOBIN=$(ROOT)/k6 go install go.k6.io/xk6/cmd/xk6@latest
|
||||||
|
cd $(ROOT)/k6 && $(ROOT)/k6/xk6 build --with github.com/phymbert/xk6-sse@latest --output $(ROOT)/k6/k6-sse
|
||||||
|
@echo "Built $(ROOT)/k6/k6-sse"
|
||||||
|
|
||||||
|
secret-commands: ## Print the exact Secret creation commands (creates nothing)
|
||||||
|
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic cloudflared \\"
|
||||||
|
@echo " --from-literal=TUNNEL_TOKEN='<cloudflare tunnel token>'"
|
||||||
|
@echo
|
||||||
|
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic control-auth \\"
|
||||||
|
@echo " --from-literal=CONTROL_USER='<control user>' \\"
|
||||||
|
@echo " --from-literal=CONTROL_PASS='<strong password>'"
|
||||||
|
@echo
|
||||||
|
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret docker-registry gitea-regcred \\"
|
||||||
|
@echo " --docker-server=$(REGISTRY) \\"
|
||||||
|
@echo " --docker-username='<gitea user>' \\"
|
||||||
|
@echo " --docker-password='<read-only package token>'"
|
||||||
|
|
||||||
|
clean: ## Remove local k6 build artifacts and results
|
||||||
|
rm -f $(ROOT)/k6/k6-sse $(ROOT)/k6/xk6
|
||||||
|
rm -rf $(K6_RESULTS)
|
||||||
|
@mkdir -p $(K6_RESULTS) && touch $(K6_RESULTS)/.gitkeep
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# PulseCast infra/QA
|
||||||
|
|
||||||
|
Namespace-only Kubernetes manifests, smoke tests, and the k6 SSE load test for
|
||||||
|
PulseCast. This repo never applies cluster changes by itself: the human runs the
|
||||||
|
commands in `docs/deploy.md`.
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
```text
|
||||||
|
deploy/k8s/ web, api, ml + cloudflared Deployments and Services, ConfigMaps
|
||||||
|
deploy/examples/ example Secret shapes (placeholders only, never applied)
|
||||||
|
scripts/smoke.sh public + in-cluster smoke checks
|
||||||
|
k6/sse.js 120 concurrent SSE clients for 5 minutes
|
||||||
|
docs/deploy.md Gate 1 runbook and exact human checklist
|
||||||
|
docs/load-test.md load-test procedure and how to record results
|
||||||
|
docs/failure-drills.md FastAPI-down, tunnel-down, pod-restart drills
|
||||||
|
Makefile validate, build, push, deploy, smoke, load helpers
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml # chmod 600
|
||||||
|
make validate # client-side dry run only, never applies
|
||||||
|
make deploy CONFIRM=yes # human only; requires the three Secrets first
|
||||||
|
make smoke # public HTTPS + API + ML health checks
|
||||||
|
make load # k6 SSE load test (needs the k6 SSE extension)
|
||||||
|
```
|
||||||
|
|
||||||
|
Read `docs/deploy.md` before the first deploy. It contains the Gitea token,
|
||||||
|
`docker login`, hello-world push, `gitea-regcred`, Cloudflare Tunnel, public
|
||||||
|
hostname, and verification steps.
|
||||||
|
|
||||||
|
## Git workflow
|
||||||
|
|
||||||
|
`main` is integration. New work happens on a `feat/<topic>` branch in its own
|
||||||
|
worktree. Before every push: `git fetch origin && git merge origin/main`.
|
||||||
|
Never force-push `main`. The initial scaffold is the one bootstrap exception
|
||||||
|
committed directly to `main`.
|
||||||
|
|
||||||
|
## Safety rules
|
||||||
|
|
||||||
|
- No Ingress, cert-manager, external-dns, Jobs, CronJobs, CRDs, Roles,
|
||||||
|
RoleBindings, NetworkPolicies, or any cluster-scoped resource.
|
||||||
|
- Exposure only through the `cloudflared` Deployment.
|
||||||
|
- No secrets, tokens, kubeconfig, or raw CSV in git.
|
||||||
|
- The human applies cluster mutations. `make deploy` refuses to run without
|
||||||
|
`CONFIRM=yes`.
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# EXAMPLE ONLY — do not apply this file.
|
||||||
|
# It exists so the manifest shapes are reviewable. Create the real secrets with
|
||||||
|
# the commands in docs/deploy.md; never commit real tokens or credentials.
|
||||||
|
#
|
||||||
|
# kubectl -n fch26-onemanarmy create secret generic cloudflared \
|
||||||
|
# --from-literal=TUNNEL_TOKEN='<cloudflare tunnel token>'
|
||||||
|
#
|
||||||
|
# kubectl -n fch26-onemanarmy create secret generic control-auth \
|
||||||
|
# --from-literal=CONTROL_USER='<control user>' \
|
||||||
|
# --from-literal=CONTROL_PASS='<strong password>'
|
||||||
|
#
|
||||||
|
# kubectl -n fch26-onemanarmy create secret docker-registry gitea-regcred \
|
||||||
|
# --docker-server=git.noelsabia.dev \
|
||||||
|
# --docker-username='<gitea user>' \
|
||||||
|
# --docker-password='<read-only package token>'
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: cloudflared
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
type: Opaque
|
||||||
|
stringData:
|
||||||
|
TUNNEL_TOKEN: "REPLACE_ME"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: control-auth
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
type: Opaque
|
||||||
|
stringData:
|
||||||
|
CONTROL_USER: "REPLACE_ME"
|
||||||
|
CONTROL_PASS: "REPLACE_ME"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: gitea-regcred
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
type: kubernetes.io/dockerconfigjson
|
||||||
|
stringData:
|
||||||
|
.dockerconfigjson: |
|
||||||
|
{"auths":{"git.noelsabia.dev":{"username":"REPLACE_ME","password":"REPLACE_ME","auth":"REPLACE_ME"}}}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
# Non-secret configuration for PulseCast.
|
||||||
|
# Namespace-scoped only. Secrets are created by the human, never committed.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: pulsecast-shared
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
data:
|
||||||
|
PUBLIC_WEB_URL: "https://pulse.noelsabia.dev"
|
||||||
|
PUBLIC_API_URL: "https://api.pulse.noelsabia.dev"
|
||||||
|
API_INTERNAL_URL: "http://api:8080"
|
||||||
|
ML_INTERNAL_URL: "http://ml:8000"
|
||||||
|
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: web-config
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: web
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
data:
|
||||||
|
NODE_ENV: "production"
|
||||||
|
HOST: "0.0.0.0"
|
||||||
|
PORT: "3000"
|
||||||
|
NUXT_TELEMETRY_DISABLED: "1"
|
||||||
|
NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev"
|
||||||
|
NUXT_API_INTERNAL_BASE: "http://api:8080"
|
||||||
|
NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: api-config
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: api
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
data:
|
||||||
|
PORT: "8080"
|
||||||
|
DATA_DIR: "/app/data"
|
||||||
|
ML_BASE_URL: "http://ml:8000"
|
||||||
|
ML_TIMEOUT: "1.5s"
|
||||||
|
SSE_HEARTBEAT: "15s"
|
||||||
|
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: ml-config
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: ml
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
data:
|
||||||
|
PORT: "8000"
|
||||||
|
DATA_DIR: "/app/data"
|
||||||
|
MODEL_PATH: "/app/model/model.joblib"
|
||||||
|
PYTHONDONTWRITEBYTECODE: "1"
|
||||||
|
PYTHONUNBUFFERED: "1"
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: ml
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: ml
|
||||||
|
app.kubernetes.io/component: ml
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
revisionHistoryLimit: 2
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: ml
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: ml
|
||||||
|
app.kubernetes.io/component: ml
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: gitea-regcred
|
||||||
|
securityContext:
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: ml
|
||||||
|
image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8000
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: ml-config
|
||||||
|
- configMapRef:
|
||||||
|
name: pulsecast-shared
|
||||||
|
env:
|
||||||
|
- name: HOME
|
||||||
|
value: /tmp
|
||||||
|
- name: MPLCONFIGDIR
|
||||||
|
value: /tmp
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
|
volumeMounts:
|
||||||
|
- name: tmp
|
||||||
|
mountPath: /tmp
|
||||||
|
startupProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: http
|
||||||
|
periodSeconds: 2
|
||||||
|
failureThreshold: 60
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /healthz
|
||||||
|
port: http
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /readyz
|
||||||
|
port: http
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 2
|
||||||
|
terminationGracePeriodSeconds: 30
|
||||||
|
volumes:
|
||||||
|
- name: tmp
|
||||||
|
emptyDir: {}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: ml
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: ml
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: ml
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 8000
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: api
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: api
|
||||||
|
app.kubernetes.io/component: backend
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
# Scenario state and the SSE hub live in memory. One replica is intentional.
|
||||||
|
replicas: 1
|
||||||
|
revisionHistoryLimit: 2
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: api
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: api
|
||||||
|
app.kubernetes.io/component: backend
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: gitea-regcred
|
||||||
|
securityContext:
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: api
|
||||||
|
image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8080
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: api-config
|
||||||
|
- configMapRef:
|
||||||
|
name: pulsecast-shared
|
||||||
|
env:
|
||||||
|
- name: HOME
|
||||||
|
value: /tmp
|
||||||
|
# Created by the human; see docs/deploy.md. Never commit real credentials.
|
||||||
|
- name: CONTROL_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: control-auth
|
||||||
|
key: CONTROL_USER
|
||||||
|
- name: CONTROL_PASS
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: control-auth
|
||||||
|
key: CONTROL_PASS
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 256Mi
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
|
volumeMounts:
|
||||||
|
- name: tmp
|
||||||
|
mountPath: /tmp
|
||||||
|
startupProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/healthz
|
||||||
|
port: http
|
||||||
|
periodSeconds: 2
|
||||||
|
failureThreshold: 30
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/healthz
|
||||||
|
port: http
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/readyz
|
||||||
|
port: http
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 2
|
||||||
|
# SSE clients reconnect with retry: 3000; give them a moment on shutdown.
|
||||||
|
terminationGracePeriodSeconds: 20
|
||||||
|
volumes:
|
||||||
|
- name: tmp
|
||||||
|
emptyDir: {}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: api
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: api
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: api
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 8080
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: web
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: web
|
||||||
|
app.kubernetes.io/component: frontend
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
revisionHistoryLimit: 2
|
||||||
|
strategy:
|
||||||
|
type: RollingUpdate
|
||||||
|
rollingUpdate:
|
||||||
|
maxUnavailable: 0
|
||||||
|
maxSurge: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: web
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: web
|
||||||
|
app.kubernetes.io/component: frontend
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
imagePullSecrets:
|
||||||
|
- name: gitea-regcred
|
||||||
|
securityContext:
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: web
|
||||||
|
# Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`.
|
||||||
|
image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3000
|
||||||
|
protocol: TCP
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: web-config
|
||||||
|
- configMapRef:
|
||||||
|
name: pulsecast-shared
|
||||||
|
env:
|
||||||
|
- name: HOME
|
||||||
|
value: /tmp
|
||||||
|
# Nuxt/Nitro may use a temp dir; keep it on the writable emptyDir.
|
||||||
|
- name: NITRO_TEMP_DIR
|
||||||
|
value: /tmp
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: 500m
|
||||||
|
memory: 512Mi
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
|
volumeMounts:
|
||||||
|
- name: tmp
|
||||||
|
mountPath: /tmp
|
||||||
|
startupProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: http
|
||||||
|
periodSeconds: 2
|
||||||
|
failureThreshold: 30
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: http
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: http
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 2
|
||||||
|
volumes:
|
||||||
|
- name: tmp
|
||||||
|
emptyDir: {}
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: web
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: web
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: web
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 3000
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
# Cloudflare Tunnel is the only public exposure path.
|
||||||
|
# Routes are configured in the Cloudflare Zero Trust dashboard:
|
||||||
|
# pulse.noelsabia.dev -> http://web:3000
|
||||||
|
# api.pulse.noelsabia.dev -> http://api:8080
|
||||||
|
# The token lives in the `cloudflared` Secret, created by the human; never in git.
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: cloudflared
|
||||||
|
namespace: fch26-onemanarmy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: cloudflared
|
||||||
|
app.kubernetes.io/component: tunnel
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
revisionHistoryLimit: 2
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: cloudflared
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: cloudflared
|
||||||
|
app.kubernetes.io/component: tunnel
|
||||||
|
app.kubernetes.io/part-of: pulsecast
|
||||||
|
spec:
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
securityContext:
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: cloudflared
|
||||||
|
# Public Docker Hub image; no Gitea pull secret needed for this pod.
|
||||||
|
image: cloudflare/cloudflared:latest
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- tunnel
|
||||||
|
- --no-autoupdate
|
||||||
|
- --metrics
|
||||||
|
- 0.0.0.0:2000
|
||||||
|
- run
|
||||||
|
- --token
|
||||||
|
- $(TUNNEL_TOKEN)
|
||||||
|
env:
|
||||||
|
- name: TUNNEL_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: cloudflared
|
||||||
|
key: TUNNEL_TOKEN
|
||||||
|
- name: HOME
|
||||||
|
value: /tmp
|
||||||
|
ports:
|
||||||
|
- name: metrics
|
||||||
|
containerPort: 2000
|
||||||
|
protocol: TCP
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 50m
|
||||||
|
memory: 64Mi
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 128Mi
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop:
|
||||||
|
- ALL
|
||||||
|
volumeMounts:
|
||||||
|
- name: tmp
|
||||||
|
mountPath: /tmp
|
||||||
|
startupProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ready
|
||||||
|
port: metrics
|
||||||
|
periodSeconds: 2
|
||||||
|
failureThreshold: 30
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ready
|
||||||
|
port: metrics
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ready
|
||||||
|
port: metrics
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 2
|
||||||
|
terminationGracePeriodSeconds: 30
|
||||||
|
volumes:
|
||||||
|
- name: tmp
|
||||||
|
emptyDir: {}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
# Failure drills
|
||||||
|
|
||||||
|
Run these **after** Gate 1 and outside the demo window. Every command is a
|
||||||
|
cluster mutation: the human runs it, not an agent. Record the result and the
|
||||||
|
recovery time. Keep `make smoke` and a `curl -N .../api/stream` open while you
|
||||||
|
drill.
|
||||||
|
|
||||||
|
## 1. FastAPI down (cached fallback)
|
||||||
|
|
||||||
|
Goal: the Go API keeps serving forecasts from baked JSON with `"cached": true`
|
||||||
|
and `/api/readyz` stays 200. FastAPI being down must not fail API readiness.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Baseline
|
||||||
|
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||||
|
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached'
|
||||||
|
|
||||||
|
# Stop ML
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=0
|
||||||
|
|
||||||
|
# Verify fallback (allow up to 2 s for the ML timeout)
|
||||||
|
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||||
|
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}'
|
||||||
|
# Expected: readyz 200; cached true; points still present
|
||||||
|
|
||||||
|
# Recover
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=1
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/ml --timeout=180s
|
||||||
|
```
|
||||||
|
|
||||||
|
Pass: no 5xx from `/api/forecast`, `cached` flips to `true` within the 1.5 s
|
||||||
|
timeout budget, and the UI shows the cached-forecast banner. Fail: a 502/503 or
|
||||||
|
an empty forecast.
|
||||||
|
|
||||||
|
## 2. Tunnel down
|
||||||
|
|
||||||
|
Goal: detect a public outage quickly, restart `cloudflared`, and verify both
|
||||||
|
public hostnames recover. Local `kubectl port-forward` is a diagnostic only,
|
||||||
|
never the public demo path.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Stop the tunnel
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/cloudflared --replicas=0
|
||||||
|
|
||||||
|
# Verify the outage from outside
|
||||||
|
curl -sS -o /dev/null -w '%{http_code}\n' https://pulse.noelsabia.dev
|
||||||
|
curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz
|
||||||
|
# Expected: 502/503/530-style edge error, not a valid 200
|
||||||
|
|
||||||
|
# Recover
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/cloudflared --replicas=1
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/cloudflared --timeout=120s
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy logs deployment/cloudflared --tail=50
|
||||||
|
# Look for: "Registered tunnel connection"
|
||||||
|
|
||||||
|
make smoke
|
||||||
|
```
|
||||||
|
|
||||||
|
If the tunnel pod is Running but the URL is still 502, check the dashboard
|
||||||
|
routes (`pulse` -> `http://web:3000`, `api.pulse` -> `http://api:8080`) and the
|
||||||
|
Secret token. A bad token keeps the pod restarting with an auth error. Rotating
|
||||||
|
the tunnel token means updating the Secret and restarting the Deployment.
|
||||||
|
|
||||||
|
## 3. Pod restart (API and SSE reconnect)
|
||||||
|
|
||||||
|
Goal: a single API pod restart causes a short 502 window, not a stuck UI. SSE
|
||||||
|
clients reconnect with `retry: 3000`; scenario state is in memory and resets to
|
||||||
|
the default, which is documented.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Watch from two terminals:
|
||||||
|
# curl -sS -N https://api.pulse.noelsabia.dev/api/stream
|
||||||
|
# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz; sleep 1; done
|
||||||
|
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy delete pod -l app.kubernetes.io/name=api
|
||||||
|
|
||||||
|
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/api --timeout=180s
|
||||||
|
curl -sS https://api.pulse.noelsabia.dev/api/healthz
|
||||||
|
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||||
|
```
|
||||||
|
|
||||||
|
Pass: the health loop shows at most a few seconds of errors, the deployment
|
||||||
|
returns to 1/1, `/api/stream` reconnects and emits a `scenario` event, and the
|
||||||
|
UI recovers without a manual reload. Note the actual outage duration in the
|
||||||
|
paper's reliability section.
|
||||||
|
|
||||||
|
Repeat for `web` (public page 502 briefly, then reloads) and `ml` (API serves
|
||||||
|
cached forecasts, no user-visible failure).
|
||||||
|
|
||||||
|
## Cleanup / invariants
|
||||||
|
|
||||||
|
- All deployments back to 1 replica.
|
||||||
|
- `make smoke` passes.
|
||||||
|
- No Ingress, Jobs, CronJobs, NetworkPolicies, or cluster-scoped resources were
|
||||||
|
created during the drills.
|
||||||
|
- Never leave `ml` or `cloudflared` scaled to zero.
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
# k6 load test
|
||||||
|
|
||||||
|
`k6/sse.js` holds 120 concurrent Server-Sent Events connections for 5 minutes
|
||||||
|
and reports connection count plus p95 event latency.
|
||||||
|
|
||||||
|
## Why a custom k6 binary
|
||||||
|
|
||||||
|
Native k6 buffers a complete HTTP response, so it cannot parse a stream that
|
||||||
|
never ends. The script imports the community `k6/x/sse` extension
|
||||||
|
(`github.com/phymbert/xk6-sse`). Newer k6 releases resolve community extensions
|
||||||
|
automatically. If the stock `grafana/k6` image rejects the import, build a
|
||||||
|
binary with the extension:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make k6-build # creates k6/k6-sse (git-ignored)
|
||||||
|
K6=./k6/k6-sse make load
|
||||||
|
```
|
||||||
|
|
||||||
|
`make load` uses, in order: `k6` on `PATH`, `k6/k6-sse`, then
|
||||||
|
`docker run --rm -i grafana/k6`.
|
||||||
|
|
||||||
|
## Procedure
|
||||||
|
|
||||||
|
1. Confirm the API is healthy and the SSE route works:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -sS https://api.pulse.noelsabia.dev/api/healthz
|
||||||
|
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream
|
||||||
|
```
|
||||||
|
|
||||||
|
You should see `retry: 3000` and a `scenario` event immediately, then a
|
||||||
|
`ping` every 15 s.
|
||||||
|
|
||||||
|
2. Run the load test:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make load
|
||||||
|
# or: API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js
|
||||||
|
```
|
||||||
|
|
||||||
|
`make load` writes `k6/results/summary-<timestamp>.json` (git-ignored).
|
||||||
|
|
||||||
|
3. During the run, post a scenario change so the scenario-event latency has
|
||||||
|
samples and the propagation path is exercised:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \
|
||||||
|
https://api.pulse.noelsabia.dev/api/scenario
|
||||||
|
```
|
||||||
|
|
||||||
|
Repeat every 30–60 s with different scenarios if you want a distribution.
|
||||||
|
Without a change, `sse_event_latency` is measured from heartbeat `ping`
|
||||||
|
events, which still proves the stream is live.
|
||||||
|
|
||||||
|
4. Record these numbers in the paper/README from the saved summary:
|
||||||
|
- `sse_connections_opened` count (must be 120)
|
||||||
|
- `sse_connections_active` max (must reach 120)
|
||||||
|
- `sse_open_success` rate (must be >= 99%)
|
||||||
|
- `sse_event_latency` p95 (target < 2000 ms)
|
||||||
|
- `sse_connection_errors` (target 0)
|
||||||
|
- `sse_events_received` total
|
||||||
|
|
||||||
|
## Pass criteria
|
||||||
|
|
||||||
|
| Metric | Target |
|
||||||
|
|---|---|
|
||||||
|
| Connections opened | 120 |
|
||||||
|
| Peak active connections | 120 |
|
||||||
|
| Open success rate | >= 99% |
|
||||||
|
| p95 event latency | < 2000 ms |
|
||||||
|
| Unexpected connection errors | 0 |
|
||||||
|
| Duration | 5 minutes |
|
||||||
|
|
||||||
|
The k6 thresholds enforce the first, third, and fourth; the others are read
|
||||||
|
from the summary. A p95 above 2 s means Cloudflare, the Go SSE hub, or the
|
||||||
|
client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s
|
||||||
|
heartbeat, and whether the connection is HTTP/2-buffered at the edge.
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- `sse_event_latency` uses the server timestamp in `data.t` (ping) or `data.at`
|
||||||
|
(scenario). It includes clock skew between the k6 host and the pod; keep both
|
||||||
|
on NTP. Negative or > 1 h values are ignored.
|
||||||
|
- Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends
|
||||||
|
cleanly instead of waiting for the 7-minute request timeout.
|
||||||
|
- One API replica means all clients share one SSE hub; a pod restart drops
|
||||||
|
every stream and clients reconnect after `retry: 3000`.
|
||||||
|
- k6 itself does not collect server CPU/memory. Use `kubectl top pods` (if
|
||||||
|
metrics-server is available) or watch the pod during the run.
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
// PulseCast SSE load test: 120 concurrent clients for 5 minutes.
|
||||||
|
//
|
||||||
|
// The native k6 http module buffers the whole response, so it cannot read
|
||||||
|
// Server-Sent Events incrementally. This script uses the community
|
||||||
|
// `k6/x/sse` extension (github.com/phymbert/xk6-sse). Recent k6 releases
|
||||||
|
// resolve `k6/x/sse` automatically; otherwise build a binary with:
|
||||||
|
// make k6-build
|
||||||
|
//
|
||||||
|
// Run:
|
||||||
|
// API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js
|
||||||
|
// make load
|
||||||
|
//
|
||||||
|
// Recorded metrics:
|
||||||
|
// sse_connections_opened counter, expected >= SSE_VUS (120)
|
||||||
|
// sse_connections_active gauge, peak should reach 120
|
||||||
|
// sse_open_success rate of VUs that opened a stream
|
||||||
|
// sse_event_latency p95 server-timestamp-to-client latency, ms
|
||||||
|
// sse_scenario_latency same, scenario events only (if any occur)
|
||||||
|
// sse_ping_latency same, heartbeat events only
|
||||||
|
// sse_events_received total events
|
||||||
|
//
|
||||||
|
// Each VU closes its stream after SSE_HOLD_PINGS heartbeats (20 * 15 s = 5 min).
|
||||||
|
// That keeps 120 streams open for the full window and closes cleanly.
|
||||||
|
import http from 'k6/http';
|
||||||
|
import sse from 'k6/x/sse';
|
||||||
|
import { check } from 'k6';
|
||||||
|
import { Counter, Gauge, Rate, Trend } from 'k6/metrics';
|
||||||
|
|
||||||
|
const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, '');
|
||||||
|
const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`;
|
||||||
|
const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120));
|
||||||
|
const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20));
|
||||||
|
const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m';
|
||||||
|
|
||||||
|
const connectionsOpened = new Counter('sse_connections_opened');
|
||||||
|
const connectionsActive = new Gauge('sse_connections_active');
|
||||||
|
const openSuccess = new Rate('sse_open_success');
|
||||||
|
const connectionErrors = new Counter('sse_connection_errors');
|
||||||
|
const eventsReceived = new Counter('sse_events_received');
|
||||||
|
const eventLatency = new Trend('sse_event_latency', true);
|
||||||
|
const scenarioLatency = new Trend('sse_scenario_latency', true);
|
||||||
|
const pingLatency = new Trend('sse_ping_latency', true);
|
||||||
|
|
||||||
|
const thresholds = {
|
||||||
|
sse_open_success: ['rate>=0.99'],
|
||||||
|
sse_event_latency: ['p(95)<2000'],
|
||||||
|
};
|
||||||
|
thresholds[`sse_connections_opened`] = [`count>=${VUS}`];
|
||||||
|
|
||||||
|
export const options = {
|
||||||
|
scenarios: {
|
||||||
|
sse_clients: {
|
||||||
|
executor: 'per-vu-iterations',
|
||||||
|
vus: VUS,
|
||||||
|
iterations: 1,
|
||||||
|
maxDuration: MAX_DURATION,
|
||||||
|
gracefulStop: '30s',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
thresholds,
|
||||||
|
// Every VU owns one long-lived connection; do not pool them.
|
||||||
|
noConnectionReuse: true,
|
||||||
|
summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max', 'count'],
|
||||||
|
};
|
||||||
|
|
||||||
|
export function setup() {
|
||||||
|
const res = http.get(`${API_URL}/api/healthz`, { tags: { endpoint: 'setup' } });
|
||||||
|
if (res.status !== 200) {
|
||||||
|
throw new Error(`API healthz returned ${res.status}; refusing to start the SSE load test`);
|
||||||
|
}
|
||||||
|
return { api: API_URL, stream: STREAM_URL, vus: VUS };
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function () {
|
||||||
|
let opened = false;
|
||||||
|
let active = false;
|
||||||
|
let closedByUs = false;
|
||||||
|
let pings = 0;
|
||||||
|
|
||||||
|
const markActive = () => {
|
||||||
|
if (!active) {
|
||||||
|
active = true;
|
||||||
|
connectionsActive.add(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const markInactive = () => {
|
||||||
|
if (active) {
|
||||||
|
active = false;
|
||||||
|
connectionsActive.add(-1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const params = {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
Accept: 'text/event-stream',
|
||||||
|
'Cache-Control': 'no-cache',
|
||||||
|
},
|
||||||
|
// Outlive the scenario; the 20th heartbeat closes the stream first.
|
||||||
|
timeout: __ENV.SSE_TIMEOUT || '7m',
|
||||||
|
tags: { endpoint: 'sse' },
|
||||||
|
};
|
||||||
|
|
||||||
|
const res = sse.open(STREAM_URL, params, function (client) {
|
||||||
|
client.on('open', function () {
|
||||||
|
opened = true;
|
||||||
|
markActive();
|
||||||
|
connectionsOpened.add(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on('event', function (event) {
|
||||||
|
eventsReceived.add(1);
|
||||||
|
const name = event.name || 'message';
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
let payload = null;
|
||||||
|
if (event.data) {
|
||||||
|
try {
|
||||||
|
payload = JSON.parse(event.data);
|
||||||
|
} catch (_) {
|
||||||
|
payload = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The contract puts the emit time in `t` (ping) or `at` (scenario).
|
||||||
|
// Latency is server timestamp -> client receipt, in milliseconds.
|
||||||
|
const stamp = payload && (payload.t || payload.at);
|
||||||
|
if (stamp) {
|
||||||
|
const emitted = Date.parse(stamp);
|
||||||
|
if (!Number.isNaN(emitted)) {
|
||||||
|
const lag = now - emitted;
|
||||||
|
// Ignore bad stamps and absurd clock skew (> 1 h).
|
||||||
|
if (lag >= 0 && lag < 3600000) {
|
||||||
|
eventLatency.add(lag);
|
||||||
|
if (name === 'scenario') scenarioLatency.add(lag);
|
||||||
|
if (name === 'ping') pingLatency.add(lag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (name === 'ping') {
|
||||||
|
pings += 1;
|
||||||
|
if (pings >= HOLD_PINGS && !closedByUs) {
|
||||||
|
closedByUs = true;
|
||||||
|
markInactive();
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on('error', function (e) {
|
||||||
|
if (!closedByUs) {
|
||||||
|
connectionErrors.add(1);
|
||||||
|
console.error(`SSE error: ${e && e.error ? e.error() : e}`);
|
||||||
|
}
|
||||||
|
markInactive();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
check(res, {
|
||||||
|
'sse status 200': (r) => r && r.status === 200,
|
||||||
|
'sse content-type': (r) =>
|
||||||
|
r && String(r.headers['Content-Type'] || '').includes('text/event-stream'),
|
||||||
|
'sse opened': () => opened,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!opened) {
|
||||||
|
connectionErrors.add(1);
|
||||||
|
}
|
||||||
|
openSuccess.add(opened);
|
||||||
|
markInactive();
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://opencode.ai/config.json",
|
||||||
|
"model": "opencode-go/deepseek-v4.1-flash"
|
||||||
|
}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# PulseCast smoke test.
|
||||||
|
#
|
||||||
|
# Defaults target the public TLS endpoints. Override for local dev:
|
||||||
|
# WEB_URL=http://localhost:3000 API_URL=http://localhost:8080 \
|
||||||
|
# ML_URL=http://localhost:8000 ./scripts/smoke.sh
|
||||||
|
#
|
||||||
|
# FastAPI is not public. If ML_URL is empty, the script uses a short
|
||||||
|
# `kubectl port-forward svc/ml` (requires KUBECONFIG). Set SKIP_ML=1 to skip.
|
||||||
|
# Set SMOKE_SSE=1 to also check that /api/stream emits an event.
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
|
||||||
|
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}"
|
||||||
|
ML_URL="${ML_URL:-}"
|
||||||
|
NS="${NS:-fch26-onemanarmy}"
|
||||||
|
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"
|
||||||
|
CURL_MAX_TIME="${CURL_MAX_TIME:-20}"
|
||||||
|
PORT_FORWARD_PORT="${PORT_FORWARD_PORT:-18000}"
|
||||||
|
SMOKE_SSE="${SMOKE_SSE:-0}"
|
||||||
|
SKIP_ML="${SKIP_ML:-0}"
|
||||||
|
|
||||||
|
passes=0
|
||||||
|
failures=0
|
||||||
|
tmpdir="$(mktemp -d)"
|
||||||
|
pf_pid=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [[ -n "$pf_pid" ]] && kill -0 "$pf_pid" 2>/dev/null; then
|
||||||
|
kill "$pf_pid" 2>/dev/null || true
|
||||||
|
wait "$pf_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -rf "$tmpdir"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
|
pass() { printf 'PASS %s\n' "$*"; passes=$((passes + 1)); }
|
||||||
|
fail() { printf 'FAIL %s\n' "$*"; failures=$((failures + 1)); }
|
||||||
|
warn() { printf 'WARN %s\n' "$*"; }
|
||||||
|
|
||||||
|
check_http() {
|
||||||
|
local name="$1" url="$2" expect="${3:-200}" body_re="${4:-}"
|
||||||
|
local body="$tmpdir/body" code
|
||||||
|
code="$(curl -sS -L --max-time "$CURL_MAX_TIME" -o "$body" -w '%{http_code}' "$url" 2>"$tmpdir/curl.err")" || code="000"
|
||||||
|
if [[ "$code" != "$expect" ]]; then
|
||||||
|
fail "$name -> HTTP $code (expected $expect): $url"
|
||||||
|
[[ -s "$tmpdir/curl.err" ]] && sed 's/^/ /' "$tmpdir/curl.err"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ -n "$body_re" ]] && ! grep -Eq "$body_re" "$body"; then
|
||||||
|
fail "$name -> HTTP $code but body did not match /$body_re/"
|
||||||
|
head -c 300 "$body" | sed 's/^/ /'
|
||||||
|
[[ -s "$body" ]] && echo
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
pass "$name -> HTTP $code"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
check_https() {
|
||||||
|
local name="$1" url="$2" verify
|
||||||
|
case "$url" in
|
||||||
|
https://*) ;;
|
||||||
|
*) warn "$name is not HTTPS ($url); TLS check skipped"; return 0 ;;
|
||||||
|
esac
|
||||||
|
verify="$(curl -sS -o /dev/null --max-time "$CURL_MAX_TIME" -w '%{ssl_verify_result}' "$url" 2>/dev/null)" || verify="error"
|
||||||
|
if [[ "$verify" == "0" ]]; then
|
||||||
|
pass "$name -> TLS verified"
|
||||||
|
else
|
||||||
|
fail "$name -> TLS verification failed (ssl_verify_result=$verify)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_ml_url() {
|
||||||
|
[[ -n "$ML_URL" ]] && return 0
|
||||||
|
command -v kubectl >/dev/null 2>&1 || return 1
|
||||||
|
|
||||||
|
local kargs=(kubectl)
|
||||||
|
[[ -n "$CONTEXT" ]] && kargs+=(--context "$CONTEXT")
|
||||||
|
echo " starting port-forward svc/ml -> 127.0.0.1:${PORT_FORWARD_PORT}" >&2
|
||||||
|
"${kargs[@]}" -n "$NS" port-forward svc/ml "${PORT_FORWARD_PORT}:8000" >"$tmpdir/pf.log" 2>&1 &
|
||||||
|
pf_pid=$!
|
||||||
|
|
||||||
|
local i
|
||||||
|
for i in $(seq 1 40); do
|
||||||
|
if curl -sS --max-time 2 -o /dev/null "http://127.0.0.1:${PORT_FORWARD_PORT}/healthz" 2>/dev/null; then
|
||||||
|
ML_URL="http://127.0.0.1:${PORT_FORWARD_PORT}"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
kill -0 "$pf_pid" 2>/dev/null || break
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
[[ -s "$tmpdir/pf.log" ]] && sed 's/^/ /' "$tmpdir/pf.log" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
check_sse() {
|
||||||
|
local out="$tmpdir/sse"
|
||||||
|
# The stream never ends by design; stop after one heartbeat interval + margin.
|
||||||
|
timeout 20 curl -sS -N --max-time 20 -H 'Accept: text/event-stream' \
|
||||||
|
"$API_URL/api/stream" >"$out" 2>"$tmpdir/sse.err" || true
|
||||||
|
if grep -Eq '^(event: (scenario|ping)|data: )' "$out"; then
|
||||||
|
pass "SSE /api/stream -> event received"
|
||||||
|
else
|
||||||
|
fail "SSE /api/stream -> no event within 20s"
|
||||||
|
head -c 300 "$out" | sed 's/^/ /'
|
||||||
|
[[ -s "$tmpdir/sse.err" ]] && sed 's/^/ /' "$tmpdir/sse.err"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
printf 'PulseCast smoke\n'
|
||||||
|
printf ' web: %s\n api: %s\n ml: %s\n\n' "$WEB_URL" "$API_URL" "${ML_URL:-<port-forward svc/ml>}"
|
||||||
|
|
||||||
|
check_http "web /" "$WEB_URL/" 200
|
||||||
|
check_https "web /" "$WEB_URL"
|
||||||
|
|
||||||
|
check_http "api /api/healthz" "$API_URL/api/healthz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
|
||||||
|
check_http "api /api/readyz" "$API_URL/api/readyz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
|
||||||
|
check_https "api /api/healthz" "$API_URL/api/healthz"
|
||||||
|
|
||||||
|
if [[ "$SKIP_ML" == "1" ]]; then
|
||||||
|
warn "ml /healthz skipped (SKIP_ML=1)"
|
||||||
|
elif resolve_ml_url; then
|
||||||
|
check_http "ml /healthz" "$ML_URL/healthz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
|
||||||
|
else
|
||||||
|
fail "ml /healthz -> cannot reach FastAPI (set ML_URL, export KUBECONFIG, or SKIP_ML=1)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$SMOKE_SSE" == "1" ]]; then
|
||||||
|
check_sse
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '\n%d passed, %d failed\n' "$passes" "$failures"
|
||||||
|
[[ "$failures" -eq 0 ]]
|
||||||
Reference in new issue
Block a user