Files
infra-k8/scripts/gate1-check.sh
T
Noel Sabia 5b199f8fe3 feat: harden Gate 1 manifests and runbook
- add read-only make gate1-check: permissions, guardrails, dry-run, Secret keys
- set numeric runAsUser/runAsGroup (SKE non-numeric user caveat), pin cloudflared and use HTTP/2
- align ML env with PULSECAST_* and wire CONTROL_TOKEN for API/ML training proxy
- fix service worktree paths and smoke.sh executable bit
- pin k6 v2.0.0 for xk6-sse, fix closed-connection metric and initial scenario latency
- rewrite docs/deploy.md as an exact copy-paste Gate 1 checklist
2026-11-14 05:10:41 +01:00

471 lines
19 KiB
Bash
Executable File

#!/usr/bin/env bash
# Read-only Gate 1 readiness check.
#
# It NEVER applies, creates, patches, deletes, scales, or restarts anything.
# Allowed calls: kubectl config/get/auth can-i/apply --dry-run=client, curl.
#
# Usage (normally through make):
# export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml
# make gate1-check
# make gate1-check CHECK_PUBLIC=1 # also curl the public URLs
# make gate1-check ALLOW_PENDING=1 # exit 0 while human secrets are pending
#
# Exit codes: 0 ready, 1 hard failure (permissions/manifests/cluster), 2 only
# human prerequisites are missing (Secrets, Deployments, public URLs).
set -uo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
NS="${NS:-fch26-onemanarmy}"
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"
REGISTRY="${REGISTRY:-git.noelsabia.dev}"
OWNER="${OWNER:-noelsabia}"
TAG="${TAG:-gate1}"
CHECK_PUBLIC="${CHECK_PUBLIC:-0}"
ALLOW_PENDING="${ALLOW_PENDING:-0}"
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}"
CURL_MAX_TIME="${CURL_MAX_TIME:-15}"
hard=0
pending=0
warnings=0
ok() { printf '[ok] %s\n' "$*"; }
bad() { printf '[FAIL] %s\n' "$*"; hard=$((hard + 1)); }
missing() { printf '[pending] %s\n' "$*"; pending=$((pending + 1)); }
warn() { printf '[warn] %s\n' "$*"; warnings=$((warnings + 1)); }
section() { printf '\n== %s\n' "$*"; }
finish() {
printf '\n== Gate 1 readiness summary\n'
printf 'hard failures: %d\n' "$hard"
printf 'pending prerequisites: %d\n' "$pending"
printf 'warnings: %d\n' "$warnings"
if ((hard > 0)); then
printf 'RESULT: NOT READY (fix the failures above)\n'
exit 1
fi
if ((pending > 0)); then
if [[ "$ALLOW_PENDING" == "1" ]]; then
printf 'RESULT: READY except for pending prerequisites (ALLOW_PENDING=1)\n'
exit 0
fi
printf 'RESULT: NOT READY (prerequisites above are still pending)\n'
exit 2
fi
printf 'RESULT: READY for Gate 1\n'
exit 0
}
trap finish EXIT
printf 'Gate 1 readiness (read-only; no cluster mutation)\n'
printf 'namespace=%s context=%s tag=%s\n' "$NS" "$CONTEXT" "$TAG"
section "kubeconfig and cluster"
if [[ -z "${KUBECONFIG:-}" ]]; then
bad "KUBECONFIG is not set"
printf ' export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml\n'
else
if [[ ! -f "$KUBECONFIG" ]]; then
bad "KUBECONFIG does not point to a file: $KUBECONFIG"
else
mode="$(stat -c '%a' "$KUBECONFIG" 2>/dev/null || echo unknown)"
if [[ "$mode" != "600" && "$mode" != "400" ]]; then
warn "kubeconfig mode is $mode; run: chmod 600 \"$KUBECONFIG\""
else
ok "kubeconfig mode $mode"
fi
fi
fi
if ! command -v kubectl >/dev/null 2>&1; then
bad "kubectl not found"
exit 1
fi
if ! kubectl config get-contexts -o name 2>/dev/null | grep -Fxq "$CONTEXT"; then
bad "context '$CONTEXT' not found in KUBECONFIG"
exit 1
fi
ok "context '$CONTEXT' exists"
KC=(kubectl --context "$CONTEXT" --namespace "$NS")
if "${KC[@]}" get --raw=/readyz --request-timeout=5s >/dev/null 2>&1; then
ok "API server reachable"
else
bad "API server is not reachable with context '$CONTEXT'"
exit 1
fi
if "${KC[@]}" get deployments >/dev/null 2>&1; then
ok "namespace '$NS' is readable"
else
bad "cannot read namespace '$NS'"
exit 1
fi
section "namespace permissions (expected values)"
check_can_i() {
local expected="$1" verb="$2" resource="$3" got
got="$("${KC[@]}" auth can-i "$verb" "$resource" 2>/dev/null)"
if [[ "$got" == "$expected" ]]; then
ok "$verb $resource -> $got"
else
bad "$verb $resource -> ${got:-<error>} (expected $expected)"
fi
}
check_can_i yes create deployments
check_can_i yes create services
check_can_i yes create configmaps
check_can_i yes create secrets
check_can_i yes create pods
check_can_i no create ingresses
check_can_i no create jobs
check_can_i no create cronjobs
check_can_i no create networkpolicies
check_can_i no create roles
check_can_i no create rolebindings
section "manifest guardrails"
if manifest_out="$(python3 - "$ROOT/deploy/k8s" "$NS" "$TAG" "$REGISTRY" "$OWNER" <<'PY' 2>&1
import glob
import os
import re
import sys
root, ns, tag, registry, owner = sys.argv[1:6]
errors = []
warns = []
docs = []
for path in sorted(glob.glob(os.path.join(root, "*.yaml"))):
with open(path, encoding="utf-8") as handle:
try:
loaded = [d for d in __import__("yaml").safe_load_all(handle) if d]
except Exception as exc: # noqa: BLE001 - report the parse error verbatim
errors.append(f"{os.path.basename(path)}: YAML parse error: {exc}")
continue
if not loaded:
errors.append(f"{os.path.basename(path)}: no YAML documents")
for doc in loaded:
doc["__file__"] = os.path.basename(path)
docs.append(doc)
if not docs:
errors.append("deploy/k8s contains no manifests")
allowed = {"ConfigMap", "Deployment", "Service"}
forbidden = {
"Ingress", "Job", "CronJob", "NetworkPolicy", "Role", "RoleBinding",
"ClusterRole", "ClusterRoleBinding", "CustomResourceDefinition",
"Namespace", "PersistentVolume", "PersistentVolumeClaim", "StorageClass",
"PodSecurityPolicy", "PriorityClass", "APIService",
"MutatingWebhookConfiguration", "ValidatingWebhookConfiguration",
}
objects = {}
configmaps = {}
services = {}
deployments = {}
for doc in docs:
where = f"{doc['__file__']}:{doc.get('kind', '?')}/{doc.get('metadata', {}).get('name', '?')}"
kind = doc.get("kind")
meta = doc.get("metadata") or {}
name = meta.get("name")
if not doc.get("apiVersion") or not kind or not name:
errors.append(f"{where}: missing apiVersion, kind, or metadata.name")
continue
if kind not in allowed:
errors.append(f"{where}: kind {kind} is not allowed (namespace-only: ConfigMap, Deployment, Service)")
continue
if kind in forbidden:
errors.append(f"{where}: forbidden kind {kind}")
if meta.get("namespace") != ns:
errors.append(f"{where}: metadata.namespace is {meta.get('namespace')!r}, expected {ns!r}")
key = (kind, name)
if key in objects:
errors.append(f"{where}: duplicate object")
objects[key] = doc
if kind == "ConfigMap":
configmaps[name] = doc
elif kind == "Service":
services[name] = doc
elif kind == "Deployment":
deployments[name] = doc
for name, cm in configmaps.items():
data = cm.get("data") or {}
for key in data:
if re.search(r"(?i)(token|password|secret|private[_-]?key|credential)", key):
errors.append(f"ConfigMap/{name}: key {key!r} looks like a secret; use a Secret created by the human")
required_config = {
"pulsecast-shared": {"PUBLIC_WEB_URL", "PUBLIC_API_URL", "API_INTERNAL_URL", "ML_INTERNAL_URL"},
"web-config": {"PORT", "NUXT_PUBLIC_API_BASE", "NUXT_API_INTERNAL_BASE"},
"api-config": {"PORT", "DATA_DIR", "FASTAPI_URL", "ML_TIMEOUT", "SSE_HEARTBEAT"},
"ml-config": {
"PORT", "PULSECAST_ROOT", "PULSECAST_ARTIFACTS", "PULSECAST_WEATHER",
"PULSECAST_MODELS", "PULSECAST_UPLOADS",
},
}
for name, keys in required_config.items():
cm = configmaps.get(name)
if cm is None:
errors.append(f"missing ConfigMap/{name}")
continue
have = set((cm.get("data") or {}).keys())
for key in sorted(keys - have):
errors.append(f"ConfigMap/{name}: missing key {key}")
expected_ports = {"web": 3000, "api": 8080, "ml": 8000, "cloudflared": 2000}
expected_uids = {"web": 10001, "api": 65532, "ml": 10001, "cloudflared": 65532}
expected_images = {
"web": f"{registry}/{owner}/pulsecast-web:{tag}",
"api": f"{registry}/{owner}/pulsecast-api:{tag}",
"ml": f"{registry}/{owner}/pulsecast-ml:{tag}",
}
gitea_images = {"web", "api", "ml"}
for name in ("web", "api", "ml", "cloudflared"):
dep = deployments.get(name)
if dep is None:
errors.append(f"missing Deployment/{name}")
continue
spec = dep.get("spec") or {}
if spec.get("replicas") != 1:
errors.append(f"Deployment/{name}: replicas should be 1")
template = (spec.get("template") or {})
pod = template.get("spec") or {}
if pod.get("automountServiceAccountToken") is not False:
errors.append(f"Deployment/{name}: automountServiceAccountToken must be false")
pod_security = pod.get("securityContext") or {}
if (pod_security.get("seccompProfile") or {}).get("type") != "RuntimeDefault":
errors.append(f"Deployment/{name}: seccompProfile must be RuntimeDefault")
for field in ("runAsUser", "runAsGroup"):
value = pod_security.get(field)
if not isinstance(value, int) or value <= 0:
errors.append(f"Deployment/{name}: {field} must be a positive numeric id (SKE runAsNonRoot caveat)")
elif value != expected_uids[name]:
errors.append(f"Deployment/{name}: {field} is {value}, expected {expected_uids[name]}")
if pod_security.get("runAsNonRoot") is True and pod_security.get("runAsUser") == 0:
errors.append(f"Deployment/{name}: runAsUser 0 conflicts with runAsNonRoot")
for field in ("hostNetwork", "hostPID", "hostIPC"):
if pod.get(field):
errors.append(f"Deployment/{name}: {field} is not allowed")
if name in gitea_images:
pulls = pod.get("imagePullSecrets") or []
if not any(p.get("name") == "gitea-regcred" for p in pulls):
errors.append(f"Deployment/{name}: imagePullSecrets must include gitea-regcred")
containers = pod.get("containers") or []
if len(containers) != 1:
errors.append(f"Deployment/{name}: expected exactly one container")
continue
container = containers[0]
if container.get("name") != name:
errors.append(f"Deployment/{name}: container name is {container.get('name')!r}")
image = container.get("image") or ""
if image.endswith(":latest"):
errors.append(f"Deployment/{name}: image tag must be pinned, not :latest")
if name in gitea_images and not image.startswith(f"{registry}/{owner}/"):
errors.append(f"Deployment/{name}: image {image!r} is not under {registry}/{owner}/")
if name in expected_images and image != expected_images[name]:
warns.append(f"Deployment/{name}: image {image!r} does not match TAG={tag}")
if name == "cloudflared" and not image.startswith("cloudflare/cloudflared:"):
errors.append(f"Deployment/{name}: unexpected image {image!r}")
if name in expected_ports:
ports = container.get("ports") or []
if not any(int(p.get("containerPort", -1)) == expected_ports[name] for p in ports):
errors.append(f"Deployment/{name}: containerPort {expected_ports[name]} not found")
for port in ports:
if port.get("hostPort"):
errors.append(f"Deployment/{name}: hostPort is not allowed")
security = container.get("securityContext") or {}
if security.get("runAsNonRoot") is not True:
errors.append(f"Deployment/{name}: runAsNonRoot must be true")
if security.get("allowPrivilegeEscalation") is not False:
errors.append(f"Deployment/{name}: allowPrivilegeEscalation must be false")
if security.get("readOnlyRootFilesystem") is not True:
errors.append(f"Deployment/{name}: readOnlyRootFilesystem must be true")
if security.get("privileged") is True:
errors.append(f"Deployment/{name}: privileged is not allowed")
if security.get("runAsUser") == 0:
errors.append(f"Deployment/{name}: runAsUser 0 is not allowed")
drops = ((security.get("capabilities") or {}).get("drop") or [])
if "ALL" not in drops:
errors.append(f"Deployment/{name}: capabilities.drop must include ALL")
resources = container.get("resources") or {}
requests = resources.get("requests") or {}
limits = resources.get("limits") or {}
for group_name, group in (("requests", requests), ("limits", limits)):
for key in ("cpu", "memory"):
if not group.get(key):
errors.append(f"Deployment/{name}: resources.{group_name}.{key} is missing")
mounts = container.get("volumeMounts") or []
tmp_mounts = [m for m in mounts if m.get("mountPath") == "/tmp"]
if not tmp_mounts:
errors.append(f"Deployment/{name}: /tmp volumeMount is missing")
else:
volume_name = tmp_mounts[0].get("name")
volumes = pod.get("volumes") or []
if not any(v.get("name") == volume_name and "emptyDir" in v for v in volumes):
errors.append(f"Deployment/{name}: /tmp must be backed by an emptyDir")
for probe in ("startupProbe", "livenessProbe", "readinessProbe"):
probe_spec = container.get(probe) or {}
http = probe_spec.get("httpGet") or {}
if not http.get("path") or not http.get("port"):
errors.append(f"Deployment/{name}: {probe}.httpGet is missing")
refs = []
for source in container.get("envFrom") or []:
if source.get("configMapRef"):
refs.append(source["configMapRef"].get("name"))
for env in container.get("env") or []:
if env.get("valueFrom", {}).get("configMapKeyRef"):
refs.append(env["valueFrom"]["configMapKeyRef"].get("name"))
for ref in refs:
if ref not in configmaps:
errors.append(f"Deployment/{name}: references missing ConfigMap/{ref}")
expected_secret_refs = {
"api": {
("control-auth", "CONTROL_USER"),
("control-auth", "CONTROL_PASS"),
("control-auth", "CONTROL_TOKEN"),
},
"ml": {("control-auth", "CONTROL_TOKEN")},
"cloudflared": {("cloudflared", "TUNNEL_TOKEN")},
}
seen_secret_refs = set()
for env in container.get("env") or []:
secret_ref = env.get("valueFrom", {}).get("secretKeyRef")
if not secret_ref:
continue
secret_name = secret_ref.get("name")
secret_key = secret_ref.get("key")
if secret_name not in ("gitea-regcred", "cloudflared", "control-auth"):
errors.append(f"Deployment/{name}: unexpected Secret reference {secret_name!r}")
seen_secret_refs.add((secret_name, secret_key))
for required in expected_secret_refs.get(name, set()):
if required not in seen_secret_refs:
errors.append(f"Deployment/{name}: missing Secret reference {required[0]}/{required[1]}")
if "cloudflared" in deployments:
args = ((deployments["cloudflared"].get("spec") or {}).get("template") or {}).get("spec", {}).get("containers", [{}])[0].get("args") or []
for required in ("tunnel", "run", "--token", "$(TUNNEL_TOKEN)", "--no-autoupdate", "--protocol", "http2"):
if required not in args:
errors.append(f"Deployment/cloudflared: args must include {required!r}")
for name in ("web", "api", "ml"):
svc = services.get(name)
if svc is None:
errors.append(f"missing Service/{name}")
continue
spec = svc.get("spec") or {}
if spec.get("type") != "ClusterIP":
errors.append(f"Service/{name}: type must be ClusterIP")
for field in ("externalIPs", "loadBalancerIP", "loadBalancerSourceRanges"):
if spec.get(field):
errors.append(f"Service/{name}: {field} is not allowed")
ports = spec.get("ports") or []
if not any(int(p.get("port", -1)) == expected_ports[name] for p in ports):
errors.append(f"Service/{name}: port {expected_ports[name]} not found")
if any(p.get("nodePort") for p in ports):
errors.append(f"Service/{name}: nodePort is not allowed")
kinds = {}
for kind, _ in objects:
kinds[kind] = kinds.get(kind, 0) + 1
summary = ", ".join(f"{kind}={count}" for kind, count in sorted(kinds.items()))
print(f"parsed {len(objects)} objects ({summary})")
for message in warns:
print(f"WARN: {message}")
for message in errors:
print(f"FAIL: {message}")
sys.exit(1 if errors else 0)
PY
)"; then
ok "static guardrails passed"
printf '%s\n' "$manifest_out" | sed 's/^/ /'
else
bad "static guardrails failed"
printf '%s\n' "$manifest_out" | sed 's/^/ /'
fi
section "kubectl client dry-run"
if dry_run_out="$("${KC[@]}" apply --dry-run=client -f "$ROOT/deploy/k8s" 2>&1)"; then
count="$(printf '%s\n' "$dry_run_out" | grep -c '(dry run)')"
ok "kubectl apply --dry-run=client passed ($count resources)"
else
bad "kubectl apply --dry-run=client failed"
printf '%s\n' "$dry_run_out" | sed 's/^/ /'
fi
section "human prerequisites: Secrets"
check_secret_keys() {
local secret="$1"
shift
if ! "${KC[@]}" get secret "$secret" >/dev/null 2>&1; then
missing "Secret/$secret is missing (create it with docs/deploy.md; never commit it)"
return
fi
ok "Secret/$secret exists"
local have key
have="$("${KC[@]}" get secret "$secret" -o go-template='{{range $k,$v := .data}}{{$k}}{{"\n"}}{{end}}' 2>/dev/null || true)"
for key in "$@"; do
if printf '%s\n' "$have" | grep -qx "$key"; then
ok "Secret/$secret has key $key"
else
missing "Secret/$secret is missing key $key"
fi
done
}
check_secret_keys gitea-regcred .dockerconfigjson
if "${KC[@]}" get secret gitea-regcred >/dev/null 2>&1; then
secret_type="$("${KC[@]}" get secret gitea-regcred -o jsonpath='{.type}' 2>/dev/null || true)"
if [[ "$secret_type" == "kubernetes.io/dockerconfigjson" ]]; then
ok "Secret/gitea-regcred type is kubernetes.io/dockerconfigjson"
else
bad "Secret/gitea-regcred type is ${secret_type:-<unknown>}, expected kubernetes.io/dockerconfigjson"
fi
fi
check_secret_keys cloudflared TUNNEL_TOKEN
check_secret_keys control-auth CONTROL_USER CONTROL_PASS CONTROL_TOKEN
section "deployment status"
for deployment in web api ml cloudflared; do
if ! "${KC[@]}" get deployment "$deployment" >/dev/null 2>&1; then
missing "Deployment/$deployment is not deployed yet"
continue
fi
ready="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.status.availableReplicas}' 2>/dev/null || true)"
desired="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
image="$("${KC[@]}" get deployment "$deployment" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true)"
if [[ -n "$ready" && -n "$desired" ]] && ((ready >= desired)); then
ok "Deployment/$deployment ready ($ready/$desired, $image)"
else
bad "Deployment/$deployment not ready (${ready:-0}/${desired:-?}, $image)"
fi
if [[ "$deployment" != "cloudflared" && "$image" != *":$TAG" ]]; then
warn "Deployment/$deployment image tag is not :$TAG"
fi
done
if [[ "$CHECK_PUBLIC" == "1" ]]; then
section "public HTTPS"
for target in "web|$WEB_URL/" "api|$API_URL/api/healthz"; do
name="${target%%|*}"
url="${target#*|}"
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "$CURL_MAX_TIME" "$url" 2>/dev/null || echo 000)"
if [[ "$code" == "200" ]]; then
ok "$name public URL -> HTTP 200 ($url)"
else
missing "$name public URL -> HTTP ${code:-000} ($url)"
fi
done
else
section "public HTTPS"
printf '[skip] CHECK_PUBLIC=0; run make smoke (or gate1-check CHECK_PUBLIC=1) after deploy\n'
fi