diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 035aaf0..92c4c81 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -14,27 +14,19 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # The runner image has python3 but neither pip nor PyYAML. - name: Install validation dependency run: | - python3 -m pip install --user pyyaml \ - || python3 -m pip install --break-system-packages pyyaml + if ! python3 -c 'import yaml' 2>/dev/null; then + apt-get update -qq + apt-get install -y -qq python3-yaml + fi - # No kubeconfig and no cluster mutation here. This is the same guardrail - # set used locally: namespace-only kinds, numeric UIDs, securityContext, - # probes, resources, env/Secret references, image tags, Service types. + # Cluster-free guardrails: namespace-only kinds, numeric UIDs, + # securityContext, probes, resources, env/Secret refs, image tags, Services. - name: Static manifest guardrails run: make gate1-check STATIC_ONLY=1 # Falls back to YAML parsing when kubectl or a cluster is unavailable. - name: YAML parse or client dry-run run: make validate - - # The brief's client dry-run, only when this runner actually has a - # reachable kubeconfig. No Secrets are configured in CI. - - name: kubectl client dry-run when a cluster is configured - run: | - if kubectl get --raw=/readyz >/dev/null 2>&1; then - kubectl apply --dry-run=client -f deploy/k8s - else - echo "No reachable kubeconfig on this runner; static guardrails already passed." - fi diff --git a/Makefile b/Makefile index 9f00afa..066a22c 100644 --- a/Makefile +++ b/Makefile @@ -43,8 +43,8 @@ PRESENTER_AT ?= 2m30s PRESENTER_TIMEOUT ?= 10s K6 ?= K6_BIN ?= $(ROOT)/k6/k6-sse -# xk6-sse v0.2.0 exposes client.on/client.close correctly on k6 v2.0.x but not -# on v2.4.0 (verified 2026-11-14). Pin the known-good k6 version. +# xk6-sse v0.2.0 works on k6 v2.0.x but not v2.4.0 (verified 2026-11-14). +# Pin the known-good k6 version. K6_BUILD_VERSION ?= v2.0.0 K6_SCRIPT := $(ROOT)/k6/sse.js K6_RESULTS := $(ROOT)/k6/results diff --git a/README.md b/README.md index 9594d90..d8323a3 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,15 @@ Makefile validate, build, push, deploy, smoke, load helpers ## Quick start +CI checks, no cluster or kubeconfig: + +```bash +make gate1-check STATIC_ONLY=1 +make validate +``` + +Full deploy, human-run: + ```bash export KUBECONFIG=/path/to/fch26-onemanarmy-sa-kubeconfig.yaml # chmod 600 make gate1-check # read-only: permissions, guardrails, dry-run, Secrets @@ -45,8 +54,8 @@ hostname, and verification steps. ## Gate 1 status -- Static `make gate1-check STATIC_ONLY=1`/CI passes. The live check correctly - flags the cluster's v1.0 images until the reviewed v1.1 branch is applied. +- CI runs the cluster-free checks (`STATIC_ONLY=1`, `make validate`) and passes; + the live check is the human-run Gate 1 check with a kubeconfig. - All three v1.1 images build and pass local manifest-equivalent runtime checks (read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker. - Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from @@ -63,15 +72,15 @@ hostname, and verification steps. retained for comparison in `docs/load-test.md`. - The ML-down/API-restart failure drill passed; details are in `docs/failure-drills.md`. -- The results documentation still needs a new PR review/merge; the v1.1 - runtime code is already merged in `main`. +- The v1.1 results are merged in `main`; this branch only updates CI and docs. ## CI `.gitea/workflows/ci.yml` runs `make gate1-check STATIC_ONLY=1` and -`make validate`. It uses no Secrets, pushes no images, and never mutates a -cluster. If the Gitea runner is unavailable, run the same two commands -locally; `STATIC_ONLY=1` needs no kubeconfig. +`make validate` on a `node:20-bookworm` runner. It installs `python3-yaml`, +uses no Secrets, pushes no images, and never touches a cluster. Run the same +two commands locally; both work without a kubeconfig but need PyYAML +(`python3-yaml`). ## Git workflow diff --git a/deploy/k8s/10-ml.yaml b/deploy/k8s/10-ml.yaml index 64dc621..aa95a97 100644 --- a/deploy/k8s/10-ml.yaml +++ b/deploy/k8s/10-ml.yaml @@ -36,8 +36,7 @@ spec: containers: - name: ml image: git.noelsabia.dev/noel/pulsecast-ml:v1.1 - # v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible. - # Switch back to IfNotPresent once the tag is frozen. + # Always makes a re-pushed v1.1 tag visible; switch to IfNotPresent once frozen. imagePullPolicy: Always ports: - name: http diff --git a/deploy/k8s/20-api.yaml b/deploy/k8s/20-api.yaml index 241606e..1dabb27 100644 --- a/deploy/k8s/20-api.yaml +++ b/deploy/k8s/20-api.yaml @@ -37,8 +37,7 @@ spec: containers: - name: api image: git.noelsabia.dev/noel/pulsecast-api:v1.1 - # v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible. - # Switch back to IfNotPresent once the tag is frozen. + # Always makes a re-pushed v1.1 tag visible; switch to IfNotPresent once frozen. imagePullPolicy: Always ports: - name: http @@ -52,7 +51,7 @@ spec: env: - name: HOME value: /tmp - # Created by the human; see docs/deploy.md. Never commit real credentials. + # Created by the human; see docs/deploy.md. - name: CONTROL_USER valueFrom: secretKeyRef: diff --git a/deploy/k8s/30-web.yaml b/deploy/k8s/30-web.yaml index 969e87d..5024cbe 100644 --- a/deploy/k8s/30-web.yaml +++ b/deploy/k8s/30-web.yaml @@ -38,10 +38,8 @@ spec: fsGroup: 10001 containers: - name: web - # Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`. image: git.noelsabia.dev/noel/pulsecast-web:v1.1 - # v1.1 may be re-pushed during the demo run-up; Always makes a re-push visible. - # Switch back to IfNotPresent once the tag is frozen. + # Always makes a re-pushed v1.1 tag visible; switch to IfNotPresent once frozen. imagePullPolicy: Always ports: - name: http diff --git a/deploy/k8s/40-cloudflared.yaml b/deploy/k8s/40-cloudflared.yaml index 3021feb..c266535 100644 --- a/deploy/k8s/40-cloudflared.yaml +++ b/deploy/k8s/40-cloudflared.yaml @@ -37,8 +37,7 @@ spec: fsGroup: 65532 containers: - name: cloudflared - # Public Docker Hub image; no Gitea pull secret needed for this pod. - # Pinned rather than :latest so a surprise upstream change cannot break Gate 1. + # Docker Hub image; pinned so an upstream change cannot break Gate 1. image: cloudflare/cloudflared:2026.11.0 imagePullPolicy: IfNotPresent args: diff --git a/docs/deploy.md b/docs/deploy.md index 91d5008..22af05c 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -9,6 +9,23 @@ agent only writes and validates manifests. No Ingress, cert-manager, external-dns, Jobs, CronJobs, CRDs, Roles, RoleBindings, NetworkPolicies, or other cluster-scoped resources are used. +## Run it + +From this repo root, with `KUBECONFIG` exported (see +[Before you start](#before-you-start)): + +```bash +make validate +make gate1-check +make build +make push +make deploy CONFIRM=yes +make smoke SMOKE_SSE=1 +``` + +The first deploy also needs the Gitea pull proof, Tunnel, and Secrets in +Steps 1–6. Every cluster mutation is human-run. + ## What Gate 1 means Gate 1 passes only when **all** of these are true: @@ -24,59 +41,13 @@ Gate 1 passes only when **all** of these are true: Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest dry-run, or a container that only runs on your laptop. -## Current external status (2026-11-14 ~08:30 CET) - -Re-check before trusting this snapshot: - -- **v1.1 is deployed and Ready:** web/api/ml `:v1.1`, cloudflared - `2026.11.0`, all four Deployments `1/1`. Numeric UIDs are - `10001:10001` for web/ml and `65532:65532` for API/cloudflared. -- **Tunnel active:** `pulsecast-gate1` serves `pulse.noelsabia.dev` through - Nuxt (`x-powered-by: Nuxt`) and `api-pulse.noelsabia.dev` through the API; - four HTTP/2 connections. `/stage` and `/live` return 200; `/control` is - 401 anonymous and 200 with control auth. The temporary hello is deleted. -- **v1.1 120-VU Cloudflare run passed:** 120/120 SSE connections opened and - closed, 0 errors, p95 event 331 ms, p95 scenario 646 ms, presenter POST - 49.60 ms, 361/361 checks. See `docs/load-test.md`. -- **Failure drill passed:** ML scaled to 0 while API readiness/forecast cache - and web stayed available; ML restored and API pod restarted cleanly. See - `docs/failure-drills.md`. -- **API/product checks:** 8 sensors, `/api/meta` `mae=3.32`, `wape=27.5`, - `r2=0.777`, cache `source=fastapi`, staffing plan under 2 s. -- **DNS works:** public resolvers return Cloudflare IPs for - `pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`. A local - `systemd-resolved` negative cache can be cleared with - `resolvectl query --cache=no NAME` followed by `resolvectl flush-caches`. -- **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only - `*.noelsabia.dev`; a two-level `api.pulse.noelsabia.dev` is not covered by - that certificate. -- **All three Secrets exist:** `gitea-regcred`, `cloudflared`, `control-auth`. -- **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the - `cloudflared` manifest uses `--protocol http2` instead of relying on - unverified UDP. - -## What is possible now vs what is blocked - -| Step | Who | Status | -|---|---|---| -| Cluster/manifest check | agent or human | **done**; static and live checks pass | -| Gitea pull + hello proof | human | **done**; hello deleted after cutover | -| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **done** | -| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active; `pulse` serves Nuxt | -| `api`, `ml`, `web` v1.1 images | human | **done**; pushed and deployed | -| v1.1 apply + smoke + presenter k6 + drills | human | **done** | -| Results docs | agent | **pending new PR review/merge** | - -`make build` and `make push` cover all three services; no Dockerfile is -missing now. Do not deploy placeholder images under the real service names. - ## Before you start -Run every block in this document in **one shell**, from this worktree. The +Run every block in this document in **one shell**, from the repo root. The commands are bash/zsh compatible. ```bash -cd /home/noel/Documents/Hackathon_2026/infra-k8.git/feat-infra-bootstrap +cd /path/to/infra-k8 export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarmy-sa-kubeconfig.yaml" export NS="fch26-onemanarmy" @@ -159,10 +130,11 @@ make validate make gate1-check ``` -Expected now (before any human work): +Expected on a fresh cluster (before any human work): -- `make validate` prints `== kubectl client dry-run (read-only)` and 11 - `created (dry run)` lines. +- With a reachable cluster, `make validate` prints + `== kubectl client dry-run (read-only)` and 11 `created (dry run)` lines. + Without one it prints `11 manifests parsed`. - `make gate1-check` reports **0 hard failures**, the expected permission matrix, and lists missing Secrets/Deployments as `[pending]`. - Exit code 2 from `make gate1-check` is correct at this stage: it means only @@ -297,11 +269,10 @@ Expected: `service/hello exposed`. ## Step 3 — Cloudflare Tunnel -The tunnel `pulsecast-gate1` already exists. `pulse.noelsabia.dev` resolves; -`api-pulse.noelsabia.dev` resolves after its route is saved. The old -`api.pulse.noelsabia.dev` also resolves but is **not certificate-safe**: the -Universal SSL certificate is `*.noelsabia.dev` and covers only one subdomain -level. Use `api-pulse.noelsabia.dev`, not `api.pulse.noelsabia.dev`. +The tunnel used here is `pulsecast-gate1`. `pulse.noelsabia.dev` resolves; +`api-pulse.noelsabia.dev` resolves after its route is saved. Universal SSL +covers only `*.noelsabia.dev`, so the API host must stay one label under the +zone: `api-pulse.noelsabia.dev`. ```bash getent hosts pulse.noelsabia.dev @@ -343,8 +314,7 @@ getent hosts api-pulse.noelsabia.dev change it to `web:3000` after the web image is deployed. If you already have the web image, you may set `web:3000` now and skip the hello route; then the public hello check in Step 6 is not available, but the tunnel proof still - happens when the real web pod is Ready. Delete the old - `api.pulse.noelsabia.dev` route if it still exists. + happens when the real web pod is Ready. 5. The hostnames must be in the Cloudflare zone for `noelsabia.dev`. Cloudflare creates the DNS records for a tunnel hostname; do **not** add @@ -440,7 +410,7 @@ test -f ../../api-go.git/main/Dockerfile && echo api-ok || echo api-blocked test -f ../../ML-py.git/main/Dockerfile && echo ml-ok || echo ml-blocked ``` -Expected **now** (all three Dockerfiles landed on 2026-11-14): +Expected: ```text web-ok @@ -466,8 +436,8 @@ constraints: - Web: image `USER 10001:10001`; with read-only root and `/tmp` tmpfs, `/`, `/stage`, and `/live` returned 200; `/control` returned 401 anonymous and 200 with control auth; the same-origin SSE proxy emitted events. -- v1.1 was pushed to Gitea. The cluster is not updated until this branch is - reviewed/merged and explicitly applied. +- v1.1 is pushed to Gitea and deployed in the cluster; re-apply only after a + reviewed image change. Push: @@ -746,6 +716,37 @@ Also save: Image digests are in the `docker push` output; tags alone are not enough if you need to reproduce the exact deploy. +## Current status (2026-11-14) + +Re-check before trusting this snapshot: + +- **v1.1 is deployed and Ready:** web/api/ml `:v1.1`, cloudflared + `2026.11.0`, all four Deployments `1/1`. Numeric UIDs are + `10001:10001` for web/ml and `65532:65532` for API/cloudflared. +- **Tunnel active:** `pulsecast-gate1` serves `pulse.noelsabia.dev` through + Nuxt (`x-powered-by: Nuxt`) and `api-pulse.noelsabia.dev` through the API; + four HTTP/2 connections. `/stage` and `/live` return 200; `/control` is + 401 anonymous and 200 with control auth. The temporary hello is deleted. +- **v1.1 120-VU Cloudflare run passed:** 120/120 SSE connections opened and + closed, 0 errors, p95 event 331 ms, p95 scenario 646 ms, presenter POST + 49.60 ms, 361/361 checks. See `docs/load-test.md`. +- **Failure drill passed:** ML scaled to 0 while API readiness/forecast cache + and web stayed available; ML restored and API pod restarted cleanly. See + `docs/failure-drills.md`. +- **API/product checks:** 8 sensors, `/api/meta` `mae=3.32`, `wape=27.5`, + `r2=0.777`, cache `source=fastapi`, staffing plan under 2 s. +- **DNS works:** public resolvers return Cloudflare IPs for + `pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`. A local + `systemd-resolved` negative cache can be cleared with + `resolvectl query --cache=no NAME` followed by `resolvectl flush-caches`. +- **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only + `*.noelsabia.dev`, so the host must stay one label under the zone. +- **All three Secrets exist:** `gitea-regcred`, `cloudflared`, `control-auth`. +- **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the + `cloudflared` manifest uses `--protocol http2` instead of relying on + unverified UDP. +- **Results docs are merged in `main`.** This branch only updates CI/docs. + ## Troubleshooting | Symptom | Check | Fix | diff --git a/docs/load-test.md b/docs/load-test.md index b3a1721..3db0efa 100644 --- a/docs/load-test.md +++ b/docs/load-test.md @@ -15,13 +15,14 @@ Build the pinned binary once: ```bash make k6-build # creates k6/k6-sse (git-ignored) -./k6/k6-sse version # expect k6 v2.0.0 + xk6-sse v0.2.0 +./k6/k6-sse version # expect k6 v2.0.0 + the current xk6-sse ``` -The build pins **k6 v2.0.0**. On k6 v2.4.0 the same `xk6-sse` v0.2.0 build -fails at runtime with `TypeError: Object has no member 'on'` (the extension's -client methods are not exposed); v2.0.0 was verified locally on 2026-11-14. If -you bump `K6_BUILD_VERSION`, rerun the script against a test stream first. +The build pins **k6 v2.0.0**; `xk6-sse` v0.2.0 is the version used for the +recorded runs. On k6 v2.4.0 that extension build fails at runtime with +`TypeError: Object has no member 'on'` (the client methods are not exposed); +v2.0.0 was verified locally on 2026-11-14. If you bump `K6_BUILD_VERSION`, +rerun the script against a test stream first. `make load` uses, in order: `K6` if set, `k6/k6-sse`, then `k6` on `PATH`. A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension. @@ -79,9 +80,9 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension. The script was validated locally on 2026-11-14 with the pinned k6 v2.0.0 build against a mock SSE stream and the real v1.1 API image (3 VUs, 1 heartbeat each): connections opened and closed cleanly, open success 100%, presenter -POST 200, and scenario latency sampled. The real 120-client run still requires -the deployed API, because localhost results say nothing about Cloudflare -buffering or SKE scheduling. +POST 200, and scenario latency sampled. Run the 120-client test against the +deployed API; localhost says nothing about Cloudflare buffering or SKE +scheduling. ## Pass criteria diff --git a/scripts/gate1-check.sh b/scripts/gate1-check.sh index 370d9f3..f7abee9 100755 --- a/scripts/gate1-check.sh +++ b/scripts/gate1-check.sh @@ -20,7 +20,7 @@ NS="${NS:-fch26-onemanarmy}" CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}" REGISTRY="${REGISTRY:-git.noelsabia.dev}" OWNER="${OWNER:-noel}" -TAG="${TAG:-gate1}" +TAG="${TAG:-v1.1}" CHECK_PUBLIC="${CHECK_PUBLIC:-0}" ALLOW_PENDING="${ALLOW_PENDING:-0}" STATIC_ONLY="${STATIC_ONLY:-0}"