From 339309cea6d9a6815224495f82f3445370719332 Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 06:18:08 +0100 Subject: [PATCH 1/5] fix: use Gitea owner noel for Gate 1 images The live hello push proved the Gitea namespace is noel, not noelsabia. - retag manifests and Makefile defaults to git.noelsabia.dev/noel/... - record the verified hello pull in the runbook - keep gate1-check defaults in sync --- Makefile | 2 +- README.md | 6 ++++-- deploy/k8s/10-ml.yaml | 2 +- deploy/k8s/20-api.yaml | 2 +- deploy/k8s/30-web.yaml | 2 +- docs/deploy.md | 23 +++++++++++++---------- scripts/gate1-check.sh | 2 +- 7 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 63d62b2..1aea9f8 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ KUBECONFORM ?= kubeconform PYTHON ?= python3 REGISTRY ?= git.noelsabia.dev -OWNER ?= noelsabia +OWNER ?= noel TAG ?= gate1 WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG) API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG) diff --git a/README.md b/README.md index 4bcb512..49ea0f0 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,10 @@ hostname, and verification steps. human prerequisites. - All three images build and pass local manifest-equivalent runtime checks (read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker. -- Still human-only: Gitea read-only token + hello image pull, `gitea-regcred`, - Cloudflare Tunnel + DNS hostnames, and the three Secrets. +- Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from + `git.noelsabia.dev/noel/hello:gate1`; the Service returned HTTP 200. +- Still human-only: Cloudflare Tunnel + DNS hostnames, `cloudflared` and + `control-auth` Secrets, then image push/deploy/smoke. ## CI diff --git a/deploy/k8s/10-ml.yaml b/deploy/k8s/10-ml.yaml index ee21180..2718ddb 100644 --- a/deploy/k8s/10-ml.yaml +++ b/deploy/k8s/10-ml.yaml @@ -35,7 +35,7 @@ spec: fsGroup: 10001 containers: - name: ml - image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1 + image: git.noelsabia.dev/noel/pulsecast-ml:gate1 # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. # Switch to an immutable tag + IfNotPresent at deploy freeze. imagePullPolicy: Always diff --git a/deploy/k8s/20-api.yaml b/deploy/k8s/20-api.yaml index fdafb5a..dcf03f0 100644 --- a/deploy/k8s/20-api.yaml +++ b/deploy/k8s/20-api.yaml @@ -36,7 +36,7 @@ spec: fsGroup: 65532 containers: - name: api - image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1 + image: git.noelsabia.dev/noel/pulsecast-api:gate1 # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. # Switch to an immutable tag + IfNotPresent at deploy freeze. imagePullPolicy: Always diff --git a/deploy/k8s/30-web.yaml b/deploy/k8s/30-web.yaml index e5f82c5..672032a 100644 --- a/deploy/k8s/30-web.yaml +++ b/deploy/k8s/30-web.yaml @@ -39,7 +39,7 @@ spec: containers: - name: web # Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`. - image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1 + image: git.noelsabia.dev/noel/pulsecast-web:gate1 # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. # Switch to an immutable tag + IfNotPresent at deploy freeze. imagePullPolicy: Always diff --git a/docs/deploy.md b/docs/deploy.md index eb1e1ff..175d075 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -35,9 +35,10 @@ Findings from `CLUSTER-SMOKE.md`; re-check before trusting this snapshot: `api.pulse.noelsabia.dev` have no records. That is expected until the Cloudflare Tunnel public hostnames are created. - **No tunnel yet:** Cloudflare Tunnel has not been created. -- **Gitea pull auth is untested:** the network path works, but no - `gitea-regcred` image pull has succeeded because the read-only token does - not exist yet. The hello pod in Step 2 is the first proof. +- **Gitea pull is proven (2026-11-14 ~06:16 CET):** `gitea-regcred` exists and + the cluster pulled `git.noelsabia.dev/noel/hello:gate1` in 120 ms; the hello + pod is `1/1 Running` and its Service returned HTTP 200. The Gitea owner is + `noel`. - **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the `cloudflared` manifest uses `--protocol http2` instead of relying on unverified UDP. @@ -54,8 +55,8 @@ Findings from `CLUSTER-SMOKE.md`; re-check before trusting this snapshot: | Step | Who | Status | |---|---|---| | Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) | -| Gitea push login + hello-world push | human | **ready now**; registry auth still untested | -| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **ready now** (needs token) | +| Gitea push login + hello-world push | human | **done**; hello pull and Service verified | +| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` **done**; `cloudflared`/`control-auth` pending | | Cloudflare Tunnel + public hostnames/DNS | human | **blocked on human** — not created; DNS absent | | `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally | | `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally | @@ -77,14 +78,16 @@ export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarm export NS="fch26-onemanarmy" export CTX="fch26-onemanarmy-sa-context" export REGISTRY="git.noelsabia.dev" -export OWNER="noelsabia" +export OWNER="noel" export TAG="gate1" chmod 600 "$KUBECONFIG" ``` The kubeconfig is **not in this repo**. If it moved, change the path; never -commit it or paste its contents anywhere. +commit it or paste its contents anywhere. The Gitea owner for this workspace +is `noel`; if your account differs, set `OWNER` and the image paths +consistently. Check the local tools: @@ -477,9 +480,9 @@ Each service image must: | Service | Image | Port | UID:GID | Health paths | |---|---|---|---|---| -| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` | -| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` | -| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` | +| web | `git.noelsabia.dev/noel/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` | +| api | `git.noelsabia.dev/noel/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` | +| ml | `git.noelsabia.dev/noel/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` | The manifests set `runAsUser`/`runAsGroup`/`fsGroup` explicitly, so the image's named user is irrelevant; the files must be readable by that numeric UID. diff --git a/scripts/gate1-check.sh b/scripts/gate1-check.sh index c544870..3355e0f 100755 --- a/scripts/gate1-check.sh +++ b/scripts/gate1-check.sh @@ -19,7 +19,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" NS="${NS:-fch26-onemanarmy}" CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}" REGISTRY="${REGISTRY:-git.noelsabia.dev}" -OWNER="${OWNER:-noelsabia}" +OWNER="${OWNER:-noel}" TAG="${TAG:-gate1}" CHECK_PUBLIC="${CHECK_PUBLIC:-0}" ALLOW_PENDING="${ALLOW_PENDING:-0}" From 2eacb6269670b94c17bb979d84dd7ead9db693fc Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 06:31:11 +0100 Subject: [PATCH 2/5] docs: match current Cloudflare Routes UI Public hostnames are under the tunnel Routes tab, not an Overview action. Record that the tunnel stays Inactive until the connector runs and that the token comes from the install command. --- docs/deploy.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 175d075..f4ed237 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -303,13 +303,25 @@ getent hosts api.pulse.noelsabia.dev || echo 'api: no DNS record yet (expected)' 1. Open Cloudflare **Zero Trust → Networks → Tunnels → Create a tunnel**. 2. Connector: **Cloudflared**. Name: `pulsecast-gate1`. -3. Save and copy the tunnel token. It is shown once. -4. Open the tunnel's **Public Hostnames** tab and add: +3. Copy the tunnel token from the connector install command. It is the long + `eyJ...` string after `--token` (or after `service install`). It is + sensitive: never paste it into chat, a commit, or a screenshot. The tunnel + stays **Inactive** with 0 routes until the connector Deployment is running; + routes can be added while it is inactive. +4. Open the tunnel and click the **Routes** tab (not Overview, and not + **Add a replica**). Click **Add route** or **Add a public hostname**, choose + **Public hostname**, and add: - | Public hostname | Service | - |---|---| - | `pulse.noelsabia.dev` | `http://hello:8080` (temporary, Gate 1-A) | - | `api.pulse.noelsabia.dev` | `http://api:8080` (works after Step 10) | + | Subdomain | Domain | Path | Service type | Service URL | + |---|---|---|---|---| + | `pulse` | `noelsabia.dev` | *(empty)* | HTTP | `hello:8080` | + | `api.pulse` | `noelsabia.dev` | *(empty)* | HTTP | `api:8080` | + + The first route is temporary for Gate 1-A. For the final Gate 1-B route, + change it to `web:3000` after the web image is deployed. If you already have + the web image, you may set `web:3000` now and skip the hello route; then the + public hello check in Step 6 is not available, but the tunnel proof still + happens when the real web pod is Ready. For the final Gate 1-B route, change `pulse.noelsabia.dev` to `http://web:3000` after the web image is deployed. If you already have the From e8e0f2035bb41d0e0a4775bb9feae988398c5ded Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 06:40:35 +0100 Subject: [PATCH 3/5] fix: use certificate-safe api-pulse hostname Cloudflare Universal SSL covers *.noelsabia.dev only one level deep. api.pulse.noelsabia.dev is two levels and fails the certificate check. - switch the API base/SSE URLs to api-pulse.noelsabia.dev - update smoke, k6, runbook, and failure drills - record the verified DNS/certificate diagnosis and local cache flush --- Makefile | 2 +- README.md | 7 +++- deploy/k8s/00-config.yaml | 6 +-- deploy/k8s/40-cloudflared.yaml | 2 +- docs/deploy.md | 70 +++++++++++++++++++--------------- docs/failure-drills.md | 20 +++++----- docs/load-test.md | 8 ++-- k6/sse.js | 4 +- scripts/gate1-check.sh | 2 +- scripts/smoke.sh | 2 +- 10 files changed, 68 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 1aea9f8..79310f5 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ API_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/feat-*)),$(fi ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main))) WEB_URL ?= https://pulse.noelsabia.dev -API_URL ?= https://api.pulse.noelsabia.dev +API_URL ?= https://api-pulse.noelsabia.dev ML_URL ?= LOAD_API_URL ?= $(API_URL) SMOKE_SSE ?= 0 diff --git a/README.md b/README.md index 49ea0f0..8d55407 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,11 @@ hostname, and verification steps. (read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker. - Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from `git.noelsabia.dev/noel/hello:gate1`; the Service returned HTTP 200. -- Still human-only: Cloudflare Tunnel + DNS hostnames, `cloudflared` and - `control-auth` Secrets, then image push/deploy/smoke. +- Tunnel `pulsecast-gate1` is created with `pulse` and `api-pulse` routes; the + connector is pending. The API hostname is `api-pulse.noelsabia.dev` because + Universal SSL only covers `*.noelsabia.dev`. +- Still needed: `cloudflared`/`control-auth` Secrets, image push, deploy, and + public smoke. ## CI diff --git a/deploy/k8s/00-config.yaml b/deploy/k8s/00-config.yaml index 010ed96..0292784 100644 --- a/deploy/k8s/00-config.yaml +++ b/deploy/k8s/00-config.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/part-of: pulsecast data: PUBLIC_WEB_URL: "https://pulse.noelsabia.dev" - PUBLIC_API_URL: "https://api.pulse.noelsabia.dev" + PUBLIC_API_URL: "https://api-pulse.noelsabia.dev" API_INTERNAL_URL: "http://api:8080" ML_INTERNAL_URL: "http://ml:8000" CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000" @@ -28,9 +28,9 @@ data: PORT: "3000" NUXT_TELEMETRY_DISABLED: "1" TMPDIR: "/tmp" - NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev" + NUXT_PUBLIC_API_BASE: "https://api-pulse.noelsabia.dev" NUXT_API_INTERNAL_BASE: "http://api:8080" - NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream" + NUXT_PUBLIC_SSE_URL: "https://api-pulse.noelsabia.dev/api/stream" --- apiVersion: v1 kind: ConfigMap diff --git a/deploy/k8s/40-cloudflared.yaml b/deploy/k8s/40-cloudflared.yaml index 53e8fbf..3021feb 100644 --- a/deploy/k8s/40-cloudflared.yaml +++ b/deploy/k8s/40-cloudflared.yaml @@ -1,7 +1,7 @@ # Cloudflare Tunnel is the only public exposure path. # Routes are configured in the Cloudflare Zero Trust dashboard: # pulse.noelsabia.dev -> http://web:3000 -# api.pulse.noelsabia.dev -> http://api:8080 +# api-pulse.noelsabia.dev -> http://api:8080 # The token lives in the `cloudflared` Secret, created by the human; never in git. apiVersion: apps/v1 kind: Deployment diff --git a/docs/deploy.md b/docs/deploy.md index f4ed237..1345958 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -19,22 +19,28 @@ Gate 1 passes only when **all** of these are true: 3. `web`, `api`, `ml`, and `cloudflared` are Ready. 4. `make smoke SMOKE_SSE=1` passes and a phone on mobile data can open `https://pulse.noelsabia.dev`. -5. `https://api.pulse.noelsabia.dev/api/healthz` returns HTTP 200. +5. `https://api-pulse.noelsabia.dev/api/healthz` returns HTTP 200. Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest dry-run, or a container that only runs on your laptop. -## Current external status (2026-11-14 03:50 UTC smoke) +## Current external status (2026-11-14 ~06:40 CET) -Findings from `CLUSTER-SMOKE.md`; re-check before trusting this snapshot: +Re-check before trusting this snapshot: - **Verified:** cluster access, scheduling, Service DNS/HTTP, probes, Secrets, non-root + read-only-root + `emptyDir`, and egress to Cloudflare, Open-Meteo, and `git.noelsabia.dev`. -- **No public DNS yet:** `pulse.noelsabia.dev` and - `api.pulse.noelsabia.dev` have no records. That is expected until the - Cloudflare Tunnel public hostnames are created. -- **No tunnel yet:** Cloudflare Tunnel has not been created. +- **Tunnel created, connector pending:** `pulsecast-gate1` has routes for + `pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`; it stays Inactive until + the `cloudflared` Deployment runs. +- **DNS works:** public resolvers return Cloudflare IPs for + `pulse.noelsabia.dev` and the old `api.pulse.noelsabia.dev`. A local + `systemd-resolved` NXDOMAIN cache can be flushed with + `resolvectl flush-caches`. +- **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only + `*.noelsabia.dev`; a two-level `api.pulse.noelsabia.dev` is not covered by + that certificate. - **Gitea pull is proven (2026-11-14 ~06:16 CET):** `gitea-regcred` exists and the cluster pulled `git.noelsabia.dev/noel/hello:gate1` in 120 ms; the hello pod is `1/1 Running` and its Service returned HTTP 200. The Gitea owner is @@ -57,11 +63,11 @@ Findings from `CLUSTER-SMOKE.md`; re-check before trusting this snapshot: | Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) | | Gitea push login + hello-world push | human | **done**; hello pull and Service verified | | `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` **done**; `cloudflared`/`control-auth` pending | -| Cloudflare Tunnel + public hostnames/DNS | human | **blocked on human** — not created; DNS absent | +| Cloudflare Tunnel + public hostnames/DNS | human | tunnel created; routes added; **connector pending** | | `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally | | `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally | | `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally | -| Full deploy + smoke + Gate 1 sign-off | human | **ready once the Gitea/Tunnel steps are done** | +| Full deploy + smoke + Gate 1 sign-off | human | **ready once the tunnel/secret/image steps are done** | `make build` and `make push` cover all three services; no Dockerfile is missing now. Do not deploy placeholder images under the real service names. @@ -145,7 +151,7 @@ Run top to bottom. Stop at the first failure and use - [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready. - [ ] 11. `make smoke SMOKE_SSE=1` passes. - [ ] 12. Phone over mobile data opens `https://pulse.noelsabia.dev`; - `https://api.pulse.noelsabia.dev/api/healthz` returns 200. + `https://api-pulse.noelsabia.dev/api/healthz` returns 200. - [ ] 13. Image tags/digests and the time are recorded. ## Step 0 — read-only readiness and dry-run @@ -293,12 +299,21 @@ Expected: `service/hello exposed`. ## Step 3 — Cloudflare Tunnel -As of the 03:50 UTC smoke, neither public hostname has a DNS record. That is -expected before the tunnel route exists; do not add manual A/AAAA records. +The tunnel `pulsecast-gate1` already exists. `pulse.noelsabia.dev` resolves; +`api-pulse.noelsabia.dev` resolves after its route is saved. The old +`api.pulse.noelsabia.dev` also resolves but is **not certificate-safe**: the +Universal SSL certificate is `*.noelsabia.dev` and covers only one subdomain +level. Use `api-pulse.noelsabia.dev`, not `api.pulse.noelsabia.dev`. ```bash -getent hosts pulse.noelsabia.dev || echo 'pulse: no DNS record yet (expected)' -getent hosts api.pulse.noelsabia.dev || echo 'api: no DNS record yet (expected)' +getent hosts pulse.noelsabia.dev +getent hosts api-pulse.noelsabia.dev || echo 'api-pulse: no DNS record yet (expected until the route exists)' +``` + +If DNS looks missing locally but public DNS has it, flush the local cache: + +```bash +resolvectl flush-caches ``` 1. Open Cloudflare **Zero Trust → Networks → Tunnels → Create a tunnel**. @@ -315,19 +330,14 @@ getent hosts api.pulse.noelsabia.dev || echo 'api: no DNS record yet (expected)' | Subdomain | Domain | Path | Service type | Service URL | |---|---|---|---|---| | `pulse` | `noelsabia.dev` | *(empty)* | HTTP | `hello:8080` | - | `api.pulse` | `noelsabia.dev` | *(empty)* | HTTP | `api:8080` | + | `api-pulse` | `noelsabia.dev` | *(empty)* | HTTP | `api:8080` | The first route is temporary for Gate 1-A. For the final Gate 1-B route, change it to `web:3000` after the web image is deployed. If you already have the web image, you may set `web:3000` now and skip the hello route; then the public hello check in Step 6 is not available, but the tunnel proof still - happens when the real web pod is Ready. - - For the final Gate 1-B route, change `pulse.noelsabia.dev` to - `http://web:3000` after the web image is deployed. If you already have the - web image, you may set `web:3000` now and skip the hello route; then the - public hello check in Step 6 is not available, but the tunnel proof still - happens when the real web pod is Ready. + happens when the real web pod is Ready. Delete the old + `api.pulse.noelsabia.dev` route if it still exists. 5. The hostnames must be in the Cloudflare zone for `noelsabia.dev`. Cloudflare creates the DNS records for a tunnel hostname; do **not** add @@ -340,7 +350,7 @@ After saving both hostnames, DNS should resolve to Cloudflare: ```bash getent hosts pulse.noelsabia.dev -getent hosts api.pulse.noelsabia.dev +getent hosts api-pulse.noelsabia.dev ``` Expected: one or more Cloudflare IPv4/IPv6 addresses. If there is still no @@ -505,7 +515,7 @@ together, and keep the value numeric. `api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`, `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and `ML_TIMEOUT`. -`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in +`web` must have `NUXT_PUBLIC_API_BASE=https://api-pulse.noelsabia.dev` baked in or read it at runtime from the ConfigMap; the browser must use the public API hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`, `PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and @@ -630,10 +640,10 @@ Manual public checks (also run from a phone on mobile data): ```bash curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/ -curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz -curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz -curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}' -curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20 +curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz +curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api-pulse.noelsabia.dev/api/readyz +curl -fsS https://api-pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}' +curl -sS -N --max-time 20 https://api-pulse.noelsabia.dev/api/stream | head -20 ``` Expected: @@ -714,11 +724,11 @@ you need to reproduce the exact deploy. | `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod -l app=hello` (and the real pod); `kubectl -n "$NS" get secret gitea-regcred` | The token must have `read:package`; the Secret must be in `fch26-onemanarmy`; the tag must exist in Gitea. Do not print the secret data. | | `CreateContainerConfigError` | `kubectl -n "$NS" describe pod ` | A referenced Secret key is missing (`control-auth`: `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`) or the `cloudflared` Secret lacks `TUNNEL_TOKEN`. Recreate the Secret. | | `container has runAsNonRoot and image will run as root` or a non-numeric-user error | `kubectl -n "$NS" get pod -o yaml`; inspect `securityContext` | Keep `runAsNonRoot: true` and set a positive numeric `runAsUser`/`runAsGroup` (SKE caveat). The manifests already do; do not "fix" this by disabling `runAsNonRoot`. | -| Public hostname has no DNS record | `getent hosts pulse.noelsabia.dev` / `getent hosts api.pulse.noelsabia.dev` | Save the public hostname in the tunnel dashboard in the `noelsabia.dev` zone; remove a conflicting A/AAAA/CNAME for that exact name. Do not add manual records for a tunnel hostname. | +| Public hostname has no DNS record | `getent hosts pulse.noelsabia.dev` / `getent hosts api-pulse.noelsabia.dev` | Save the public hostname in the tunnel dashboard in the `noelsabia.dev` zone; remove a conflicting A/AAAA/CNAME for that exact name. Do not add manual records for a tunnel hostname. | | `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. | | `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. | | Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. | -| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api.pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. | +| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. | | `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. | | `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. | | SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. | diff --git a/docs/failure-drills.md b/docs/failure-drills.md index 321e98b..d4bf299 100644 --- a/docs/failure-drills.md +++ b/docs/failure-drills.md @@ -12,15 +12,15 @@ and `/api/readyz` stays 200. FastAPI being down must not fail API readiness. ```bash # Baseline -curl -sS https://api.pulse.noelsabia.dev/api/readyz -curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached' +curl -sS https://api-pulse.noelsabia.dev/api/readyz +curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached' # Stop ML kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=0 # Verify fallback (allow up to 2 s for the ML timeout) -curl -sS https://api.pulse.noelsabia.dev/api/readyz -curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}' +curl -sS https://api-pulse.noelsabia.dev/api/readyz +curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}' # Expected: readyz 200; cached true; points still present # Recover @@ -44,7 +44,7 @@ kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployme # Verify the outage from outside curl -sS -o /dev/null -w '%{http_code}\n' https://pulse.noelsabia.dev -curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz +curl -sS -o /dev/null -w '%{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz # Expected: 502/503/530-style edge error, not a valid 200 # Recover @@ -57,7 +57,7 @@ make smoke ``` If the tunnel pod is Running but the URL is still 502, check the dashboard -routes (`pulse` -> `http://web:3000`, `api.pulse` -> `http://api:8080`) and the +routes (`pulse` -> `http://web:3000`, `api-pulse` -> `http://api:8080`) and the Secret token. A bad token keeps the pod restarting with an auth error. Rotating the tunnel token means updating the Secret and restarting the Deployment. @@ -69,14 +69,14 @@ the default, which is documented. ```bash # Watch from two terminals: -# curl -sS -N https://api.pulse.noelsabia.dev/api/stream -# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz; sleep 1; done +# curl -sS -N https://api-pulse.noelsabia.dev/api/stream +# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz; sleep 1; done kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy delete pod -l app.kubernetes.io/name=api kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/api --timeout=180s -curl -sS https://api.pulse.noelsabia.dev/api/healthz -curl -sS https://api.pulse.noelsabia.dev/api/readyz +curl -sS https://api-pulse.noelsabia.dev/api/healthz +curl -sS https://api-pulse.noelsabia.dev/api/readyz ``` Pass: the health loop shows at most a few seconds of errors, the deployment diff --git a/docs/load-test.md b/docs/load-test.md index b56d9ef..4ecd75d 100644 --- a/docs/load-test.md +++ b/docs/load-test.md @@ -29,8 +29,8 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension. 1. Confirm the API is healthy and the SSE route works: ```bash - curl -sS https://api.pulse.noelsabia.dev/api/healthz - curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream + curl -sS https://api-pulse.noelsabia.dev/api/healthz + curl -sS -N --max-time 20 https://api-pulse.noelsabia.dev/api/stream ``` You should see `retry: 3000` and a `scenario` event immediately, then a @@ -40,7 +40,7 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension. ```bash make load - # or: API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js + # or: API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js ``` `make load` writes `k6/results/summary-.json` (git-ignored). @@ -52,7 +52,7 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension. curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \ -H 'Content-Type: application/json' \ -d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \ - https://api.pulse.noelsabia.dev/api/scenario + https://api-pulse.noelsabia.dev/api/scenario ``` Repeat every 30–60 s with different scenarios if you want a distribution. diff --git a/k6/sse.js b/k6/sse.js index 3aefcc4..cfc5f90 100644 --- a/k6/sse.js +++ b/k6/sse.js @@ -7,7 +7,7 @@ // make k6-build // // Run: -// API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js +// API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js // make load // // Recorded metrics: @@ -26,7 +26,7 @@ import sse from 'k6/x/sse'; import { check } from 'k6'; import { Counter, Rate, Trend } from 'k6/metrics'; -const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, ''); +const API_URL = (__ENV.API_URL || 'https://api-pulse.noelsabia.dev').replace(/\/+$/, ''); const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`; const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120)); const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20)); diff --git a/scripts/gate1-check.sh b/scripts/gate1-check.sh index 3355e0f..2959db6 100755 --- a/scripts/gate1-check.sh +++ b/scripts/gate1-check.sh @@ -25,7 +25,7 @@ CHECK_PUBLIC="${CHECK_PUBLIC:-0}" ALLOW_PENDING="${ALLOW_PENDING:-0}" STATIC_ONLY="${STATIC_ONLY:-0}" WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}" -API_URL="${API_URL:-https://api.pulse.noelsabia.dev}" +API_URL="${API_URL:-https://api-pulse.noelsabia.dev}" CURL_MAX_TIME="${CURL_MAX_TIME:-15}" hard=0 diff --git a/scripts/smoke.sh b/scripts/smoke.sh index eaab5ff..37ca1fc 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -11,7 +11,7 @@ set -uo pipefail WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}" -API_URL="${API_URL:-https://api.pulse.noelsabia.dev}" +API_URL="${API_URL:-https://api-pulse.noelsabia.dev}" ML_URL="${ML_URL:-}" NS="${NS:-fch26-onemanarmy}" CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}" From eed7e62aa7b7c978c052b859c92d6ec54509b3ca Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 06:47:05 +0100 Subject: [PATCH 4/5] docs: note systemd-resolved negative cache recovery --- docs/deploy.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/deploy.md b/docs/deploy.md index 1345958..3852aed 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -316,6 +316,15 @@ If DNS looks missing locally but public DNS has it, flush the local cache: resolvectl flush-caches ``` +If `getent` still misses the name while `dig +short NAME @1.1.1.1` works, force +a fresh upstream query and flush again: + +```bash +resolvectl query --cache=no api-pulse.noelsabia.dev +resolvectl flush-caches +getent hosts api-pulse.noelsabia.dev +``` + 1. Open Cloudflare **Zero Trust → Networks → Tunnels → Create a tunnel**. 2. Connector: **Cloudflared**. Name: `pulsecast-gate1`. 3. Copy the tunnel token from the connector install command. It is the long From 40bc8e47d059a2974e6b90968e054074be509c4d Mon Sep 17 00:00:00 2001 From: Noel Sabia Date: Sat, 14 Nov 2026 06:51:06 +0100 Subject: [PATCH 5/5] docs: record active tunnel and public hello --- README.md | 10 +++++----- docs/deploy.md | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 8d55407..c88db03 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,11 @@ hostname, and verification steps. (read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker. - Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from `git.noelsabia.dev/noel/hello:gate1`; the Service returned HTTP 200. -- Tunnel `pulsecast-gate1` is created with `pulse` and `api-pulse` routes; the - connector is pending. The API hostname is `api-pulse.noelsabia.dev` because - Universal SSL only covers `*.noelsabia.dev`. -- Still needed: `cloudflared`/`control-auth` Secrets, image push, deploy, and - public smoke. +- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes; + `https://pulse.noelsabia.dev` returns 200 from hello with valid TLS. The API + hostname is `api-pulse.noelsabia.dev` because Universal SSL only covers + `*.noelsabia.dev`. +- Still needed: `control-auth` Secret, image push, deploy, and public smoke. ## CI diff --git a/docs/deploy.md b/docs/deploy.md index 3852aed..7edf019 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -24,20 +24,21 @@ Gate 1 passes only when **all** of these are true: Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest dry-run, or a container that only runs on your laptop. -## Current external status (2026-11-14 ~06:40 CET) +## Current external status (2026-11-14 ~06:50 CET) Re-check before trusting this snapshot: - **Verified:** cluster access, scheduling, Service DNS/HTTP, probes, Secrets, non-root + read-only-root + `emptyDir`, and egress to Cloudflare, Open-Meteo, and `git.noelsabia.dev`. -- **Tunnel created, connector pending:** `pulsecast-gate1` has routes for - `pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`; it stays Inactive until - the `cloudflared` Deployment runs. +- **Tunnel active:** `pulsecast-gate1` has routes for `pulse.noelsabia.dev` + and `api-pulse.noelsabia.dev`; `cloudflared` is `1/1` with four HTTP/2 + connections, and `https://pulse.noelsabia.dev` returns 200 from hello with + valid TLS. `api-pulse` returns 502 until the API is deployed. - **DNS works:** public resolvers return Cloudflare IPs for - `pulse.noelsabia.dev` and the old `api.pulse.noelsabia.dev`. A local - `systemd-resolved` NXDOMAIN cache can be flushed with - `resolvectl flush-caches`. + `pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`. A local + `systemd-resolved` negative cache can be cleared with + `resolvectl query --cache=no NAME` followed by `resolvectl flush-caches`. - **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only `*.noelsabia.dev`; a two-level `api.pulse.noelsabia.dev` is not covered by that certificate. @@ -51,10 +52,10 @@ Re-check before trusting this snapshot: - **Numeric UIDs are required:** a non-numeric image user breaks `runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in `deploy/k8s/` sets explicit numeric `runAsUser`/`runAsGroup`/`fsGroup`. -- **Repo status (2026-11-14 ~05:10 UTC):** all three images now build and pass +- **Repo status (2026-11-14 ~06:50 CET):** all three images build and pass local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp` - tmpfs). There is no remaining Dockerfile blocker; the human Gitea/Tunnel - steps are what remain. + tmpfs). Remaining steps are the image push, `control-auth`, deploy, and + public smoke. ## What is possible now vs what is blocked @@ -62,8 +63,8 @@ Re-check before trusting this snapshot: |---|---|---| | Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) | | Gitea push login + hello-world push | human | **done**; hello pull and Service verified | -| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` **done**; `cloudflared`/`control-auth` pending | -| Cloudflare Tunnel + public hostnames/DNS | human | tunnel created; routes added; **connector pending** | +| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` + `cloudflared` **done**; `control-auth` pending | +| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active, public hello 200 | | `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally | | `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally | | `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally |