chore: bootstrap infra-k8 scaffold
This commit is contained in:
commit
aeee20cb7f
14 files changed
+1256
No files matched your search
+17
@@ -0,0 +1,17 @@
|
||||
# Local secrets and cluster credentials
|
||||
*kubeconfig*
|
||||
*.kubeconfig
|
||||
.env
|
||||
.env.*
|
||||
secrets.local.yaml
|
||||
deploy/secrets*.yaml
|
||||
|
||||
# k6 binaries and run output
|
||||
k6/k6-sse
|
||||
k6/xk6
|
||||
k6/results/*
|
||||
!k6/results/.gitkeep
|
||||
|
||||
# Editor / OS noise
|
||||
.DS_Store
|
||||
*.swp
|
||||
@@ -0,0 +1,153 @@
|
||||
SHELL := /usr/bin/env bash
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
|
||||
|
||||
NAMESPACE ?= fch26-onemanarmy
|
||||
CONTEXT ?= fch26-onemanarmy-sa-context
|
||||
KUBECTL ?= kubectl --context $(CONTEXT) --namespace $(NAMESPACE)
|
||||
|
||||
REGISTRY ?= git.noelsabia.dev
|
||||
OWNER ?= noelsabia
|
||||
TAG ?= gate1
|
||||
WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
|
||||
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
|
||||
ML_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-ml:$(TAG)
|
||||
|
||||
WEB_DIR ?= $(ROOT)/../web-nuxt
|
||||
API_DIR ?= $(ROOT)/../api-go
|
||||
ML_DIR ?= $(ROOT)/../ML-py
|
||||
|
||||
WEB_URL ?= https://pulse.noelsabia.dev
|
||||
API_URL ?= https://api.pulse.noelsabia.dev
|
||||
LOAD_API_URL ?= $(API_URL)
|
||||
SSE_VUS ?= 120
|
||||
SSE_HOLD_PINGS ?= 20
|
||||
SSE_MAX_DURATION ?= 6m
|
||||
K6_IMAGE ?= grafana/k6:latest
|
||||
K6_SCRIPT := $(ROOT)/k6/sse.js
|
||||
K6_RESULTS := $(ROOT)/k6/results
|
||||
|
||||
.PHONY: help validate validate-kubeconform build build-web build-api build-ml push push-web push-api push-ml set-image deploy deploy-check status logs smoke smoke-local load k6-build secret-commands clean
|
||||
|
||||
help: ## Show available targets
|
||||
@awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z0-9_-]+:.*## / {printf " %-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
||||
|
||||
validate: ## Client-side dry-run of deploy/k8s (never applies)
|
||||
@command -v kubectl >/dev/null 2>&1 || { echo "kubectl not found"; exit 1; }
|
||||
kubectl apply --dry-run=client --validate=false -f $(ROOT)/deploy/k8s
|
||||
@if command -v kubeconform >/dev/null 2>&1; then \
|
||||
kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s; \
|
||||
else \
|
||||
echo "kubeconform not installed; kubectl client dry-run is the schema check"; \
|
||||
fi
|
||||
|
||||
validate-kubeconform: ## Validate schemas with kubeconform if installed
|
||||
kubeconform -strict -summary -ignore-missing-schemas $(ROOT)/deploy/k8s
|
||||
|
||||
build: build-web build-api build-ml ## Build all three images locally
|
||||
|
||||
build-web: ## Build the web image
|
||||
@test -f $(WEB_DIR)/Dockerfile || { echo "missing $(WEB_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
|
||||
docker build -t $(WEB_IMAGE) $(WEB_DIR)
|
||||
|
||||
build-api: ## Build the api image
|
||||
@test -f $(API_DIR)/Dockerfile || { echo "missing $(API_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
|
||||
docker build -t $(API_IMAGE) $(API_DIR)
|
||||
|
||||
build-ml: ## Build the ml image
|
||||
@test -f $(ML_DIR)/Dockerfile || { echo "missing $(ML_DIR)/Dockerfile (coordinate through the orchestrator)"; exit 1; }
|
||||
docker build -t $(ML_IMAGE) $(ML_DIR)
|
||||
|
||||
push: push-web push-api push-ml ## Push all three images to Gitea
|
||||
|
||||
push-web: ## Push the web image
|
||||
docker push $(WEB_IMAGE)
|
||||
|
||||
push-api: ## Push the api image
|
||||
docker push $(API_IMAGE)
|
||||
|
||||
push-ml: ## Push the ml image
|
||||
docker push $(ML_IMAGE)
|
||||
|
||||
set-image: ## Update live Deployments to TAG (guarded; human only)
|
||||
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
|
||||
$(KUBECTL) set image deployment/web web=$(WEB_IMAGE)
|
||||
$(KUBECTL) set image deployment/api api=$(API_IMAGE)
|
||||
$(KUBECTL) set image deployment/ml ml=$(ML_IMAGE)
|
||||
$(KUBECTL) rollout status deployment/web deployment/api deployment/ml --timeout=180s
|
||||
|
||||
deploy-check: ## Verify required Secrets exist (read-only)
|
||||
@for s in gitea-regcred cloudflared control-auth; do \
|
||||
$(KUBECTL) get secret "$$s" >/dev/null 2>&1 || { \
|
||||
echo "Missing Secret $$s in $(NAMESPACE); see docs/deploy.md"; exit 1; }; \
|
||||
done
|
||||
@echo "required Secrets present"
|
||||
|
||||
deploy: deploy-check ## Apply manifests (guarded; human only)
|
||||
@test "$(CONFIRM)" = "yes" || { echo "Refusing: re-run with CONFIRM=yes (cluster mutation)"; exit 1; }
|
||||
$(KUBECTL) apply -f $(ROOT)/deploy/k8s
|
||||
$(KUBECTL) rollout status deployment/ml --timeout=180s
|
||||
$(KUBECTL) rollout status deployment/api --timeout=180s
|
||||
$(KUBECTL) rollout status deployment/web --timeout=180s
|
||||
$(KUBECTL) rollout status deployment/cloudflared --timeout=120s
|
||||
@echo
|
||||
@echo "Deployed. Now run: make smoke"
|
||||
|
||||
status: ## Show deployments, pods, services, and secrets
|
||||
$(KUBECTL) get deploy,pods,svc,secrets
|
||||
|
||||
logs: ## Follow cloudflared logs
|
||||
$(KUBECTL) logs -f deployment/cloudflared
|
||||
|
||||
smoke: ## Public HTTPS + API + ML health checks
|
||||
WEB_URL="$(WEB_URL)" API_URL="$(API_URL)" NS="$(NAMESPACE)" CONTEXT="$(CONTEXT)" \
|
||||
$(ROOT)/scripts/smoke.sh
|
||||
|
||||
smoke-local: ## Smoke the three local dev servers
|
||||
WEB_URL="http://localhost:3000" API_URL="http://localhost:8080" \
|
||||
ML_URL="http://localhost:8000" $(ROOT)/scripts/smoke.sh
|
||||
|
||||
load: ## Run the 120-client / 5-minute SSE load test
|
||||
@mkdir -p $(K6_RESULTS)
|
||||
@export API_URL="$(LOAD_API_URL)" SSE_VUS="$(SSE_VUS)" SSE_HOLD_PINGS="$(SSE_HOLD_PINGS)" \
|
||||
SSE_MAX_DURATION="$(SSE_MAX_DURATION)"; \
|
||||
stamp=$$(date +%Y%m%dT%H%M%S); \
|
||||
if command -v k6 >/dev/null 2>&1; then \
|
||||
echo "Using k6 from PATH"; \
|
||||
k6 run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \
|
||||
elif [ -x "$(ROOT)/k6/k6-sse" ]; then \
|
||||
echo "Using $(ROOT)/k6/k6-sse"; \
|
||||
"$(ROOT)/k6/k6-sse" run --summary-export="$(K6_RESULTS)/summary-$$stamp.json" "$(K6_SCRIPT)"; \
|
||||
else \
|
||||
echo "Using docker image $(K6_IMAGE)"; \
|
||||
docker run --rm -i \
|
||||
-v "$(ROOT)/k6:/scripts:ro" \
|
||||
-v "$(K6_RESULTS):/results" \
|
||||
-e API_URL -e SSE_VUS -e SSE_HOLD_PINGS -e SSE_MAX_DURATION \
|
||||
$(K6_IMAGE) run --summary-export="/results/summary-$$stamp.json" /scripts/sse.js; \
|
||||
fi
|
||||
|
||||
k6-build: ## Build a k6 binary with the SSE extension (k6/k6-sse)
|
||||
@mkdir -p $(K6_RESULTS)
|
||||
cd $(ROOT)/k6 && GOBIN=$(ROOT)/k6 go install go.k6.io/xk6/cmd/xk6@latest
|
||||
cd $(ROOT)/k6 && $(ROOT)/k6/xk6 build --with github.com/phymbert/xk6-sse@latest --output $(ROOT)/k6/k6-sse
|
||||
@echo "Built $(ROOT)/k6/k6-sse"
|
||||
|
||||
secret-commands: ## Print the exact Secret creation commands (creates nothing)
|
||||
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic cloudflared \\"
|
||||
@echo " --from-literal=TUNNEL_TOKEN='<cloudflare tunnel token>'"
|
||||
@echo
|
||||
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret generic control-auth \\"
|
||||
@echo " --from-literal=CONTROL_USER='<control user>' \\"
|
||||
@echo " --from-literal=CONTROL_PASS='<strong password>'"
|
||||
@echo
|
||||
@echo "kubectl --context $(CONTEXT) -n $(NAMESPACE) create secret docker-registry gitea-regcred \\"
|
||||
@echo " --docker-server=$(REGISTRY) \\"
|
||||
@echo " --docker-username='<gitea user>' \\"
|
||||
@echo " --docker-password='<read-only package token>'"
|
||||
|
||||
clean: ## Remove local k6 build artifacts and results
|
||||
rm -f $(ROOT)/k6/k6-sse $(ROOT)/k6/xk6
|
||||
rm -rf $(K6_RESULTS)
|
||||
@mkdir -p $(K6_RESULTS) && touch $(K6_RESULTS)/.gitkeep
|
||||
@@ -0,0 +1,48 @@
|
||||
# PulseCast infra/QA
|
||||
|
||||
Namespace-only Kubernetes manifests, smoke tests, and the k6 SSE load test for
|
||||
PulseCast. This repo never applies cluster changes by itself: the human runs the
|
||||
commands in `docs/deploy.md`.
|
||||
|
||||
## Layout
|
||||
|
||||
```text
|
||||
deploy/k8s/ web, api, ml + cloudflared Deployments and Services, ConfigMaps
|
||||
deploy/examples/ example Secret shapes (placeholders only, never applied)
|
||||
scripts/smoke.sh public + in-cluster smoke checks
|
||||
k6/sse.js 120 concurrent SSE clients for 5 minutes
|
||||
docs/deploy.md Gate 1 runbook and exact human checklist
|
||||
docs/load-test.md load-test procedure and how to record results
|
||||
docs/failure-drills.md FastAPI-down, tunnel-down, pod-restart drills
|
||||
Makefile validate, build, push, deploy, smoke, load helpers
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml # chmod 600
|
||||
make validate # client-side dry run only, never applies
|
||||
make deploy CONFIRM=yes # human only; requires the three Secrets first
|
||||
make smoke # public HTTPS + API + ML health checks
|
||||
make load # k6 SSE load test (needs the k6 SSE extension)
|
||||
```
|
||||
|
||||
Read `docs/deploy.md` before the first deploy. It contains the Gitea token,
|
||||
`docker login`, hello-world push, `gitea-regcred`, Cloudflare Tunnel, public
|
||||
hostname, and verification steps.
|
||||
|
||||
## Git workflow
|
||||
|
||||
`main` is integration. New work happens on a `feat/<topic>` branch in its own
|
||||
worktree. Before every push: `git fetch origin && git merge origin/main`.
|
||||
Never force-push `main`. The initial scaffold is the one bootstrap exception
|
||||
committed directly to `main`.
|
||||
|
||||
## Safety rules
|
||||
|
||||
- No Ingress, cert-manager, external-dns, Jobs, CronJobs, CRDs, Roles,
|
||||
RoleBindings, NetworkPolicies, or any cluster-scoped resource.
|
||||
- Exposure only through the `cloudflared` Deployment.
|
||||
- No secrets, tokens, kubeconfig, or raw CSV in git.
|
||||
- The human applies cluster mutations. `make deploy` refuses to run without
|
||||
`CONFIRM=yes`.
|
||||
@@ -0,0 +1,43 @@
|
||||
# EXAMPLE ONLY — do not apply this file.
|
||||
# It exists so the manifest shapes are reviewable. Create the real secrets with
|
||||
# the commands in docs/deploy.md; never commit real tokens or credentials.
|
||||
#
|
||||
# kubectl -n fch26-onemanarmy create secret generic cloudflared \
|
||||
# --from-literal=TUNNEL_TOKEN='<cloudflare tunnel token>'
|
||||
#
|
||||
# kubectl -n fch26-onemanarmy create secret generic control-auth \
|
||||
# --from-literal=CONTROL_USER='<control user>' \
|
||||
# --from-literal=CONTROL_PASS='<strong password>'
|
||||
#
|
||||
# kubectl -n fch26-onemanarmy create secret docker-registry gitea-regcred \
|
||||
# --docker-server=git.noelsabia.dev \
|
||||
# --docker-username='<gitea user>' \
|
||||
# --docker-password='<read-only package token>'
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: cloudflared
|
||||
namespace: fch26-onemanarmy
|
||||
type: Opaque
|
||||
stringData:
|
||||
TUNNEL_TOKEN: "REPLACE_ME"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: control-auth
|
||||
namespace: fch26-onemanarmy
|
||||
type: Opaque
|
||||
stringData:
|
||||
CONTROL_USER: "REPLACE_ME"
|
||||
CONTROL_PASS: "REPLACE_ME"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: gitea-regcred
|
||||
namespace: fch26-onemanarmy
|
||||
type: kubernetes.io/dockerconfigjson
|
||||
stringData:
|
||||
.dockerconfigjson: |
|
||||
{"auths":{"git.noelsabia.dev":{"username":"REPLACE_ME","password":"REPLACE_ME","auth":"REPLACE_ME"}}}
|
||||
@@ -0,0 +1,63 @@
|
||||
# Non-secret configuration for PulseCast.
|
||||
# Namespace-scoped only. Secrets are created by the human, never committed.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: pulsecast-shared
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
data:
|
||||
PUBLIC_WEB_URL: "https://pulse.noelsabia.dev"
|
||||
PUBLIC_API_URL: "https://api.pulse.noelsabia.dev"
|
||||
API_INTERNAL_URL: "http://api:8080"
|
||||
ML_INTERNAL_URL: "http://ml:8000"
|
||||
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: web-config
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: web
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
data:
|
||||
NODE_ENV: "production"
|
||||
HOST: "0.0.0.0"
|
||||
PORT: "3000"
|
||||
NUXT_TELEMETRY_DISABLED: "1"
|
||||
NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev"
|
||||
NUXT_API_INTERNAL_BASE: "http://api:8080"
|
||||
NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: api-config
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: api
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
data:
|
||||
PORT: "8080"
|
||||
DATA_DIR: "/app/data"
|
||||
ML_BASE_URL: "http://ml:8000"
|
||||
ML_TIMEOUT: "1.5s"
|
||||
SSE_HEARTBEAT: "15s"
|
||||
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: ml-config
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: ml
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
data:
|
||||
PORT: "8000"
|
||||
DATA_DIR: "/app/data"
|
||||
MODEL_PATH: "/app/model/model.joblib"
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
PYTHONUNBUFFERED: "1"
|
||||
@@ -0,0 +1,107 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ml
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: ml
|
||||
app.kubernetes.io/component: ml
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 2
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ml
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: ml
|
||||
app.kubernetes.io/component: ml
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
imagePullSecrets:
|
||||
- name: gitea-regcred
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: ml
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8000
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: ml-config
|
||||
- configMapRef:
|
||||
name: pulsecast-shared
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
- name: MPLCONFIGDIR
|
||||
value: /tmp
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
periodSeconds: 2
|
||||
failureThreshold: 60
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 2
|
||||
terminationGracePeriodSeconds: 30
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ml
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: ml
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: ml
|
||||
ports:
|
||||
- name: http
|
||||
port: 8000
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,118 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: api
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: api
|
||||
app.kubernetes.io/component: backend
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
# Scenario state and the SSE hub live in memory. One replica is intentional.
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 2
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: api
|
||||
app.kubernetes.io/component: backend
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
imagePullSecrets:
|
||||
- name: gitea-regcred
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: api
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: api-config
|
||||
- configMapRef:
|
||||
name: pulsecast-shared
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
# Created by the human; see docs/deploy.md. Never commit real credentials.
|
||||
- name: CONTROL_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: control-auth
|
||||
key: CONTROL_USER
|
||||
- name: CONTROL_PASS
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: control-auth
|
||||
key: CONTROL_PASS
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /api/healthz
|
||||
port: http
|
||||
periodSeconds: 2
|
||||
failureThreshold: 30
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/healthz
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/readyz
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 2
|
||||
# SSE clients reconnect with retry: 3000; give them a moment on shutdown.
|
||||
terminationGracePeriodSeconds: 20
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: api
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: api
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: api
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,111 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: web
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: web
|
||||
app.kubernetes.io/component: frontend
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 2
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: web
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: web
|
||||
app.kubernetes.io/component: frontend
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
imagePullSecrets:
|
||||
- name: gitea-regcred
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: web
|
||||
# Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`.
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
protocol: TCP
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: web-config
|
||||
- configMapRef:
|
||||
name: pulsecast-shared
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
# Nuxt/Nitro may use a temp dir; keep it on the writable emptyDir.
|
||||
- name: NITRO_TEMP_DIR
|
||||
value: /tmp
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
periodSeconds: 2
|
||||
failureThreshold: 30
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 2
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: web
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: web
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: web
|
||||
ports:
|
||||
- name: http
|
||||
port: 3000
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
@@ -0,0 +1,99 @@
|
||||
# Cloudflare Tunnel is the only public exposure path.
|
||||
# Routes are configured in the Cloudflare Zero Trust dashboard:
|
||||
# pulse.noelsabia.dev -> http://web:3000
|
||||
# api.pulse.noelsabia.dev -> http://api:8080
|
||||
# The token lives in the `cloudflared` Secret, created by the human; never in git.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: cloudflared
|
||||
namespace: fch26-onemanarmy
|
||||
labels:
|
||||
app.kubernetes.io/name: cloudflared
|
||||
app.kubernetes.io/component: tunnel
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 2
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: cloudflared
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: cloudflared
|
||||
app.kubernetes.io/component: tunnel
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: cloudflared
|
||||
# Public Docker Hub image; no Gitea pull secret needed for this pod.
|
||||
image: cloudflare/cloudflared:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- tunnel
|
||||
- --no-autoupdate
|
||||
- --metrics
|
||||
- 0.0.0.0:2000
|
||||
- run
|
||||
- --token
|
||||
- $(TUNNEL_TOKEN)
|
||||
env:
|
||||
- name: TUNNEL_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: cloudflared
|
||||
key: TUNNEL_TOKEN
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 2000
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: metrics
|
||||
periodSeconds: 2
|
||||
failureThreshold: 30
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: metrics
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: metrics
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 2
|
||||
terminationGracePeriodSeconds: 30
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
@@ -0,0 +1,96 @@
|
||||
# Failure drills
|
||||
|
||||
Run these **after** Gate 1 and outside the demo window. Every command is a
|
||||
cluster mutation: the human runs it, not an agent. Record the result and the
|
||||
recovery time. Keep `make smoke` and a `curl -N .../api/stream` open while you
|
||||
drill.
|
||||
|
||||
## 1. FastAPI down (cached fallback)
|
||||
|
||||
Goal: the Go API keeps serving forecasts from baked JSON with `"cached": true`
|
||||
and `/api/readyz` stays 200. FastAPI being down must not fail API readiness.
|
||||
|
||||
```bash
|
||||
# Baseline
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached'
|
||||
|
||||
# Stop ML
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=0
|
||||
|
||||
# Verify fallback (allow up to 2 s for the ML timeout)
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}'
|
||||
# Expected: readyz 200; cached true; points still present
|
||||
|
||||
# Recover
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=1
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/ml --timeout=180s
|
||||
```
|
||||
|
||||
Pass: no 5xx from `/api/forecast`, `cached` flips to `true` within the 1.5 s
|
||||
timeout budget, and the UI shows the cached-forecast banner. Fail: a 502/503 or
|
||||
an empty forecast.
|
||||
|
||||
## 2. Tunnel down
|
||||
|
||||
Goal: detect a public outage quickly, restart `cloudflared`, and verify both
|
||||
public hostnames recover. Local `kubectl port-forward` is a diagnostic only,
|
||||
never the public demo path.
|
||||
|
||||
```bash
|
||||
# Stop the tunnel
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/cloudflared --replicas=0
|
||||
|
||||
# Verify the outage from outside
|
||||
curl -sS -o /dev/null -w '%{http_code}\n' https://pulse.noelsabia.dev
|
||||
curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz
|
||||
# Expected: 502/503/530-style edge error, not a valid 200
|
||||
|
||||
# Recover
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/cloudflared --replicas=1
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/cloudflared --timeout=120s
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy logs deployment/cloudflared --tail=50
|
||||
# Look for: "Registered tunnel connection"
|
||||
|
||||
make smoke
|
||||
```
|
||||
|
||||
If the tunnel pod is Running but the URL is still 502, check the dashboard
|
||||
routes (`pulse` -> `http://web:3000`, `api.pulse` -> `http://api:8080`) and the
|
||||
Secret token. A bad token keeps the pod restarting with an auth error. Rotating
|
||||
the tunnel token means updating the Secret and restarting the Deployment.
|
||||
|
||||
## 3. Pod restart (API and SSE reconnect)
|
||||
|
||||
Goal: a single API pod restart causes a short 502 window, not a stuck UI. SSE
|
||||
clients reconnect with `retry: 3000`; scenario state is in memory and resets to
|
||||
the default, which is documented.
|
||||
|
||||
```bash
|
||||
# Watch from two terminals:
|
||||
# curl -sS -N https://api.pulse.noelsabia.dev/api/stream
|
||||
# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz; sleep 1; done
|
||||
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy delete pod -l app.kubernetes.io/name=api
|
||||
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/api --timeout=180s
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/healthz
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||
```
|
||||
|
||||
Pass: the health loop shows at most a few seconds of errors, the deployment
|
||||
returns to 1/1, `/api/stream` reconnects and emits a `scenario` event, and the
|
||||
UI recovers without a manual reload. Note the actual outage duration in the
|
||||
paper's reliability section.
|
||||
|
||||
Repeat for `web` (public page 502 briefly, then reloads) and `ml` (API serves
|
||||
cached forecasts, no user-visible failure).
|
||||
|
||||
## Cleanup / invariants
|
||||
|
||||
- All deployments back to 1 replica.
|
||||
- `make smoke` passes.
|
||||
- No Ingress, Jobs, CronJobs, NetworkPolicies, or cluster-scoped resources were
|
||||
created during the drills.
|
||||
- Never leave `ml` or `cloudflared` scaled to zero.
|
||||
@@ -0,0 +1,91 @@
|
||||
# k6 load test
|
||||
|
||||
`k6/sse.js` holds 120 concurrent Server-Sent Events connections for 5 minutes
|
||||
and reports connection count plus p95 event latency.
|
||||
|
||||
## Why a custom k6 binary
|
||||
|
||||
Native k6 buffers a complete HTTP response, so it cannot parse a stream that
|
||||
never ends. The script imports the community `k6/x/sse` extension
|
||||
(`github.com/phymbert/xk6-sse`). Newer k6 releases resolve community extensions
|
||||
automatically. If the stock `grafana/k6` image rejects the import, build a
|
||||
binary with the extension:
|
||||
|
||||
```bash
|
||||
make k6-build # creates k6/k6-sse (git-ignored)
|
||||
K6=./k6/k6-sse make load
|
||||
```
|
||||
|
||||
`make load` uses, in order: `k6` on `PATH`, `k6/k6-sse`, then
|
||||
`docker run --rm -i grafana/k6`.
|
||||
|
||||
## Procedure
|
||||
|
||||
1. Confirm the API is healthy and the SSE route works:
|
||||
|
||||
```bash
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/healthz
|
||||
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream
|
||||
```
|
||||
|
||||
You should see `retry: 3000` and a `scenario` event immediately, then a
|
||||
`ping` every 15 s.
|
||||
|
||||
2. Run the load test:
|
||||
|
||||
```bash
|
||||
make load
|
||||
# or: API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js
|
||||
```
|
||||
|
||||
`make load` writes `k6/results/summary-<timestamp>.json` (git-ignored).
|
||||
|
||||
3. During the run, post a scenario change so the scenario-event latency has
|
||||
samples and the propagation path is exercised:
|
||||
|
||||
```bash
|
||||
curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \
|
||||
https://api.pulse.noelsabia.dev/api/scenario
|
||||
```
|
||||
|
||||
Repeat every 30–60 s with different scenarios if you want a distribution.
|
||||
Without a change, `sse_event_latency` is measured from heartbeat `ping`
|
||||
events, which still proves the stream is live.
|
||||
|
||||
4. Record these numbers in the paper/README from the saved summary:
|
||||
- `sse_connections_opened` count (must be 120)
|
||||
- `sse_connections_active` max (must reach 120)
|
||||
- `sse_open_success` rate (must be >= 99%)
|
||||
- `sse_event_latency` p95 (target < 2000 ms)
|
||||
- `sse_connection_errors` (target 0)
|
||||
- `sse_events_received` total
|
||||
|
||||
## Pass criteria
|
||||
|
||||
| Metric | Target |
|
||||
|---|---|
|
||||
| Connections opened | 120 |
|
||||
| Peak active connections | 120 |
|
||||
| Open success rate | >= 99% |
|
||||
| p95 event latency | < 2000 ms |
|
||||
| Unexpected connection errors | 0 |
|
||||
| Duration | 5 minutes |
|
||||
|
||||
The k6 thresholds enforce the first, third, and fourth; the others are read
|
||||
from the summary. A p95 above 2 s means Cloudflare, the Go SSE hub, or the
|
||||
client is buffering/delaying events; check `X-Accel-Buffering: no`, the 15 s
|
||||
heartbeat, and whether the connection is HTTP/2-buffered at the edge.
|
||||
|
||||
## Notes
|
||||
|
||||
- `sse_event_latency` uses the server timestamp in `data.t` (ping) or `data.at`
|
||||
(scenario). It includes clock skew between the k6 host and the pod; keep both
|
||||
on NTP. Negative or > 1 h values are ignored.
|
||||
- Each VU closes after 20 heartbeats (20 × 15 s = 5 min), so the run ends
|
||||
cleanly instead of waiting for the 7-minute request timeout.
|
||||
- One API replica means all clients share one SSE hub; a pod restart drops
|
||||
every stream and clients reconnect after `retry: 3000`.
|
||||
- k6 itself does not collect server CPU/memory. Use `kubectl top pods` (if
|
||||
metrics-server is available) or watch the pod during the run.
|
||||
@@ -0,0 +1,172 @@
|
||||
// PulseCast SSE load test: 120 concurrent clients for 5 minutes.
|
||||
//
|
||||
// The native k6 http module buffers the whole response, so it cannot read
|
||||
// Server-Sent Events incrementally. This script uses the community
|
||||
// `k6/x/sse` extension (github.com/phymbert/xk6-sse). Recent k6 releases
|
||||
// resolve `k6/x/sse` automatically; otherwise build a binary with:
|
||||
// make k6-build
|
||||
//
|
||||
// Run:
|
||||
// API_URL=https://api.pulse.noelsabia.dev k6 run k6/sse.js
|
||||
// make load
|
||||
//
|
||||
// Recorded metrics:
|
||||
// sse_connections_opened counter, expected >= SSE_VUS (120)
|
||||
// sse_connections_active gauge, peak should reach 120
|
||||
// sse_open_success rate of VUs that opened a stream
|
||||
// sse_event_latency p95 server-timestamp-to-client latency, ms
|
||||
// sse_scenario_latency same, scenario events only (if any occur)
|
||||
// sse_ping_latency same, heartbeat events only
|
||||
// sse_events_received total events
|
||||
//
|
||||
// Each VU closes its stream after SSE_HOLD_PINGS heartbeats (20 * 15 s = 5 min).
|
||||
// That keeps 120 streams open for the full window and closes cleanly.
|
||||
import http from 'k6/http';
|
||||
import sse from 'k6/x/sse';
|
||||
import { check } from 'k6';
|
||||
import { Counter, Gauge, Rate, Trend } from 'k6/metrics';
|
||||
|
||||
const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, '');
|
||||
const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`;
|
||||
const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120));
|
||||
const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20));
|
||||
const MAX_DURATION = __ENV.SSE_MAX_DURATION || '6m';
|
||||
|
||||
const connectionsOpened = new Counter('sse_connections_opened');
|
||||
const connectionsActive = new Gauge('sse_connections_active');
|
||||
const openSuccess = new Rate('sse_open_success');
|
||||
const connectionErrors = new Counter('sse_connection_errors');
|
||||
const eventsReceived = new Counter('sse_events_received');
|
||||
const eventLatency = new Trend('sse_event_latency', true);
|
||||
const scenarioLatency = new Trend('sse_scenario_latency', true);
|
||||
const pingLatency = new Trend('sse_ping_latency', true);
|
||||
|
||||
const thresholds = {
|
||||
sse_open_success: ['rate>=0.99'],
|
||||
sse_event_latency: ['p(95)<2000'],
|
||||
};
|
||||
thresholds[`sse_connections_opened`] = [`count>=${VUS}`];
|
||||
|
||||
export const options = {
|
||||
scenarios: {
|
||||
sse_clients: {
|
||||
executor: 'per-vu-iterations',
|
||||
vus: VUS,
|
||||
iterations: 1,
|
||||
maxDuration: MAX_DURATION,
|
||||
gracefulStop: '30s',
|
||||
},
|
||||
},
|
||||
thresholds,
|
||||
// Every VU owns one long-lived connection; do not pool them.
|
||||
noConnectionReuse: true,
|
||||
summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max', 'count'],
|
||||
};
|
||||
|
||||
export function setup() {
|
||||
const res = http.get(`${API_URL}/api/healthz`, { tags: { endpoint: 'setup' } });
|
||||
if (res.status !== 200) {
|
||||
throw new Error(`API healthz returned ${res.status}; refusing to start the SSE load test`);
|
||||
}
|
||||
return { api: API_URL, stream: STREAM_URL, vus: VUS };
|
||||
}
|
||||
|
||||
export default function () {
|
||||
let opened = false;
|
||||
let active = false;
|
||||
let closedByUs = false;
|
||||
let pings = 0;
|
||||
|
||||
const markActive = () => {
|
||||
if (!active) {
|
||||
active = true;
|
||||
connectionsActive.add(1);
|
||||
}
|
||||
};
|
||||
const markInactive = () => {
|
||||
if (active) {
|
||||
active = false;
|
||||
connectionsActive.add(-1);
|
||||
}
|
||||
};
|
||||
|
||||
const params = {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
},
|
||||
// Outlive the scenario; the 20th heartbeat closes the stream first.
|
||||
timeout: __ENV.SSE_TIMEOUT || '7m',
|
||||
tags: { endpoint: 'sse' },
|
||||
};
|
||||
|
||||
const res = sse.open(STREAM_URL, params, function (client) {
|
||||
client.on('open', function () {
|
||||
opened = true;
|
||||
markActive();
|
||||
connectionsOpened.add(1);
|
||||
});
|
||||
|
||||
client.on('event', function (event) {
|
||||
eventsReceived.add(1);
|
||||
const name = event.name || 'message';
|
||||
const now = Date.now();
|
||||
|
||||
let payload = null;
|
||||
if (event.data) {
|
||||
try {
|
||||
payload = JSON.parse(event.data);
|
||||
} catch (_) {
|
||||
payload = null;
|
||||
}
|
||||
}
|
||||
|
||||
// The contract puts the emit time in `t` (ping) or `at` (scenario).
|
||||
// Latency is server timestamp -> client receipt, in milliseconds.
|
||||
const stamp = payload && (payload.t || payload.at);
|
||||
if (stamp) {
|
||||
const emitted = Date.parse(stamp);
|
||||
if (!Number.isNaN(emitted)) {
|
||||
const lag = now - emitted;
|
||||
// Ignore bad stamps and absurd clock skew (> 1 h).
|
||||
if (lag >= 0 && lag < 3600000) {
|
||||
eventLatency.add(lag);
|
||||
if (name === 'scenario') scenarioLatency.add(lag);
|
||||
if (name === 'ping') pingLatency.add(lag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (name === 'ping') {
|
||||
pings += 1;
|
||||
if (pings >= HOLD_PINGS && !closedByUs) {
|
||||
closedByUs = true;
|
||||
markInactive();
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
client.on('error', function (e) {
|
||||
if (!closedByUs) {
|
||||
connectionErrors.add(1);
|
||||
console.error(`SSE error: ${e && e.error ? e.error() : e}`);
|
||||
}
|
||||
markInactive();
|
||||
});
|
||||
});
|
||||
|
||||
check(res, {
|
||||
'sse status 200': (r) => r && r.status === 200,
|
||||
'sse content-type': (r) =>
|
||||
r && String(r.headers['Content-Type'] || '').includes('text/event-stream'),
|
||||
'sse opened': () => opened,
|
||||
});
|
||||
|
||||
if (!opened) {
|
||||
connectionErrors.add(1);
|
||||
}
|
||||
openSuccess.add(opened);
|
||||
markInactive();
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"model": "opencode-go/deepseek-v4.1-flash"
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
# PulseCast smoke test.
|
||||
#
|
||||
# Defaults target the public TLS endpoints. Override for local dev:
|
||||
# WEB_URL=http://localhost:3000 API_URL=http://localhost:8080 \
|
||||
# ML_URL=http://localhost:8000 ./scripts/smoke.sh
|
||||
#
|
||||
# FastAPI is not public. If ML_URL is empty, the script uses a short
|
||||
# `kubectl port-forward svc/ml` (requires KUBECONFIG). Set SKIP_ML=1 to skip.
|
||||
# Set SMOKE_SSE=1 to also check that /api/stream emits an event.
|
||||
set -uo pipefail
|
||||
|
||||
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
|
||||
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}"
|
||||
ML_URL="${ML_URL:-}"
|
||||
NS="${NS:-fch26-onemanarmy}"
|
||||
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-20}"
|
||||
PORT_FORWARD_PORT="${PORT_FORWARD_PORT:-18000}"
|
||||
SMOKE_SSE="${SMOKE_SSE:-0}"
|
||||
SKIP_ML="${SKIP_ML:-0}"
|
||||
|
||||
passes=0
|
||||
failures=0
|
||||
tmpdir="$(mktemp -d)"
|
||||
pf_pid=""
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "$pf_pid" ]] && kill -0 "$pf_pid" 2>/dev/null; then
|
||||
kill "$pf_pid" 2>/dev/null || true
|
||||
wait "$pf_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
pass() { printf 'PASS %s\n' "$*"; passes=$((passes + 1)); }
|
||||
fail() { printf 'FAIL %s\n' "$*"; failures=$((failures + 1)); }
|
||||
warn() { printf 'WARN %s\n' "$*"; }
|
||||
|
||||
check_http() {
|
||||
local name="$1" url="$2" expect="${3:-200}" body_re="${4:-}"
|
||||
local body="$tmpdir/body" code
|
||||
code="$(curl -sS -L --max-time "$CURL_MAX_TIME" -o "$body" -w '%{http_code}' "$url" 2>"$tmpdir/curl.err")" || code="000"
|
||||
if [[ "$code" != "$expect" ]]; then
|
||||
fail "$name -> HTTP $code (expected $expect): $url"
|
||||
[[ -s "$tmpdir/curl.err" ]] && sed 's/^/ /' "$tmpdir/curl.err"
|
||||
return 1
|
||||
fi
|
||||
if [[ -n "$body_re" ]] && ! grep -Eq "$body_re" "$body"; then
|
||||
fail "$name -> HTTP $code but body did not match /$body_re/"
|
||||
head -c 300 "$body" | sed 's/^/ /'
|
||||
[[ -s "$body" ]] && echo
|
||||
return 1
|
||||
fi
|
||||
pass "$name -> HTTP $code"
|
||||
return 0
|
||||
}
|
||||
|
||||
check_https() {
|
||||
local name="$1" url="$2" verify
|
||||
case "$url" in
|
||||
https://*) ;;
|
||||
*) warn "$name is not HTTPS ($url); TLS check skipped"; return 0 ;;
|
||||
esac
|
||||
verify="$(curl -sS -o /dev/null --max-time "$CURL_MAX_TIME" -w '%{ssl_verify_result}' "$url" 2>/dev/null)" || verify="error"
|
||||
if [[ "$verify" == "0" ]]; then
|
||||
pass "$name -> TLS verified"
|
||||
else
|
||||
fail "$name -> TLS verification failed (ssl_verify_result=$verify)"
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_ml_url() {
|
||||
[[ -n "$ML_URL" ]] && return 0
|
||||
command -v kubectl >/dev/null 2>&1 || return 1
|
||||
|
||||
local kargs=(kubectl)
|
||||
[[ -n "$CONTEXT" ]] && kargs+=(--context "$CONTEXT")
|
||||
echo " starting port-forward svc/ml -> 127.0.0.1:${PORT_FORWARD_PORT}" >&2
|
||||
"${kargs[@]}" -n "$NS" port-forward svc/ml "${PORT_FORWARD_PORT}:8000" >"$tmpdir/pf.log" 2>&1 &
|
||||
pf_pid=$!
|
||||
|
||||
local i
|
||||
for i in $(seq 1 40); do
|
||||
if curl -sS --max-time 2 -o /dev/null "http://127.0.0.1:${PORT_FORWARD_PORT}/healthz" 2>/dev/null; then
|
||||
ML_URL="http://127.0.0.1:${PORT_FORWARD_PORT}"
|
||||
return 0
|
||||
fi
|
||||
kill -0 "$pf_pid" 2>/dev/null || break
|
||||
sleep 0.5
|
||||
done
|
||||
[[ -s "$tmpdir/pf.log" ]] && sed 's/^/ /' "$tmpdir/pf.log" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
check_sse() {
|
||||
local out="$tmpdir/sse"
|
||||
# The stream never ends by design; stop after one heartbeat interval + margin.
|
||||
timeout 20 curl -sS -N --max-time 20 -H 'Accept: text/event-stream' \
|
||||
"$API_URL/api/stream" >"$out" 2>"$tmpdir/sse.err" || true
|
||||
if grep -Eq '^(event: (scenario|ping)|data: )' "$out"; then
|
||||
pass "SSE /api/stream -> event received"
|
||||
else
|
||||
fail "SSE /api/stream -> no event within 20s"
|
||||
head -c 300 "$out" | sed 's/^/ /'
|
||||
[[ -s "$tmpdir/sse.err" ]] && sed 's/^/ /' "$tmpdir/sse.err"
|
||||
fi
|
||||
}
|
||||
|
||||
printf 'PulseCast smoke\n'
|
||||
printf ' web: %s\n api: %s\n ml: %s\n\n' "$WEB_URL" "$API_URL" "${ML_URL:-<port-forward svc/ml>}"
|
||||
|
||||
check_http "web /" "$WEB_URL/" 200
|
||||
check_https "web /" "$WEB_URL"
|
||||
|
||||
check_http "api /api/healthz" "$API_URL/api/healthz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
|
||||
check_http "api /api/readyz" "$API_URL/api/readyz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
|
||||
check_https "api /api/healthz" "$API_URL/api/healthz"
|
||||
|
||||
if [[ "$SKIP_ML" == "1" ]]; then
|
||||
warn "ml /healthz skipped (SKIP_ML=1)"
|
||||
elif resolve_ml_url; then
|
||||
check_http "ml /healthz" "$ML_URL/healthz" 200 '"ok"[[:space:]]*:[[:space:]]*true'
|
||||
else
|
||||
fail "ml /healthz -> cannot reach FastAPI (set ML_URL, export KUBECONFIG, or SKIP_ML=1)"
|
||||
fi
|
||||
|
||||
if [[ "$SMOKE_SSE" == "1" ]]; then
|
||||
check_sse
|
||||
fi
|
||||
|
||||
printf '\n%d passed, %d failed\n' "$passes" "$failures"
|
||||
[[ "$failures" -eq 0 ]]
|
||||
Reference in new issue
Block a user