Merge pull request 'feat: infra bootstrap' (#1) from feat/infra-bootstrap into main
infra-ci / manifests (push) Failing after 4s

Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
Noel committed 2026-11-14 06:08:27 +00:00
commit 8d993c28a6
13 files changed
+115 -75

No files matched your search

+2 -2
View File
@@ -10,7 +10,7 @@ KUBECONFORM ?= kubeconform
PYTHON ?= python3 PYTHON ?= python3
REGISTRY ?= git.noelsabia.dev REGISTRY ?= git.noelsabia.dev
OWNER ?= noelsabia OWNER ?= noel
TAG ?= gate1 TAG ?= gate1
WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG) WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG) API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
@@ -27,7 +27,7 @@ API_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/feat-*)),$(fi
ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main))) ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main)))
WEB_URL ?= https://pulse.noelsabia.dev WEB_URL ?= https://pulse.noelsabia.dev
API_URL ?= https://api.pulse.noelsabia.dev API_URL ?= https://api-pulse.noelsabia.dev
ML_URL ?= ML_URL ?=
LOAD_API_URL ?= $(API_URL) LOAD_API_URL ?= $(API_URL)
SMOKE_SSE ?= 0 SMOKE_SSE ?= 0
+7 -2
View File
@@ -49,8 +49,13 @@ hostname, and verification steps.
human prerequisites. human prerequisites.
- All three images build and pass local manifest-equivalent runtime checks - All three images build and pass local manifest-equivalent runtime checks
(read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker. (read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker.
- Still human-only: Gitea read-only token + hello image pull, `gitea-regcred`, - Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from
Cloudflare Tunnel + DNS hostnames, and the three Secrets. `git.noelsabia.dev/noel/hello:gate1`; the Service returned HTTP 200.
- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes;
`https://pulse.noelsabia.dev` returns 200 from hello with valid TLS. The API
hostname is `api-pulse.noelsabia.dev` because Universal SSL only covers
`*.noelsabia.dev`.
- Still needed: `control-auth` Secret, image push, deploy, and public smoke.
## CI ## CI
+3 -3
View File
@@ -9,7 +9,7 @@ metadata:
app.kubernetes.io/part-of: pulsecast app.kubernetes.io/part-of: pulsecast
data: data:
PUBLIC_WEB_URL: "https://pulse.noelsabia.dev" PUBLIC_WEB_URL: "https://pulse.noelsabia.dev"
PUBLIC_API_URL: "https://api.pulse.noelsabia.dev" PUBLIC_API_URL: "https://api-pulse.noelsabia.dev"
API_INTERNAL_URL: "http://api:8080" API_INTERNAL_URL: "http://api:8080"
ML_INTERNAL_URL: "http://ml:8000" ML_INTERNAL_URL: "http://ml:8000"
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000" CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
@@ -28,9 +28,9 @@ data:
PORT: "3000" PORT: "3000"
NUXT_TELEMETRY_DISABLED: "1" NUXT_TELEMETRY_DISABLED: "1"
TMPDIR: "/tmp" TMPDIR: "/tmp"
NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev" NUXT_PUBLIC_API_BASE: "https://api-pulse.noelsabia.dev"
NUXT_API_INTERNAL_BASE: "http://api:8080" NUXT_API_INTERNAL_BASE: "http://api:8080"
NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream" NUXT_PUBLIC_SSE_URL: "https://api-pulse.noelsabia.dev/api/stream"
--- ---
apiVersion: v1 apiVersion: v1
kind: ConfigMap kind: ConfigMap
+1 -1
View File
@@ -35,7 +35,7 @@ spec:
fsGroup: 10001 fsGroup: 10001
containers: containers:
- name: ml - name: ml
image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1 image: git.noelsabia.dev/noel/pulsecast-ml:gate1
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
# Switch to an immutable tag + IfNotPresent at deploy freeze. # Switch to an immutable tag + IfNotPresent at deploy freeze.
imagePullPolicy: Always imagePullPolicy: Always
+1 -1
View File
@@ -36,7 +36,7 @@ spec:
fsGroup: 65532 fsGroup: 65532
containers: containers:
- name: api - name: api
image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1 image: git.noelsabia.dev/noel/pulsecast-api:gate1
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
# Switch to an immutable tag + IfNotPresent at deploy freeze. # Switch to an immutable tag + IfNotPresent at deploy freeze.
imagePullPolicy: Always imagePullPolicy: Always
+1 -1
View File
@@ -39,7 +39,7 @@ spec:
containers: containers:
- name: web - name: web
# Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`. # Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`.
image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1 image: git.noelsabia.dev/noel/pulsecast-web:gate1
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible. # Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
# Switch to an immutable tag + IfNotPresent at deploy freeze. # Switch to an immutable tag + IfNotPresent at deploy freeze.
imagePullPolicy: Always imagePullPolicy: Always
+1 -1
View File
@@ -1,7 +1,7 @@
# Cloudflare Tunnel is the only public exposure path. # Cloudflare Tunnel is the only public exposure path.
# Routes are configured in the Cloudflare Zero Trust dashboard: # Routes are configured in the Cloudflare Zero Trust dashboard:
# pulse.noelsabia.dev -> http://web:3000 # pulse.noelsabia.dev -> http://web:3000
# api.pulse.noelsabia.dev -> http://api:8080 # api-pulse.noelsabia.dev -> http://api:8080
# The token lives in the `cloudflared` Secret, created by the human; never in git. # The token lives in the `cloudflared` Secret, created by the human; never in git.
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
+80 -45
View File
@@ -19,48 +19,56 @@ Gate 1 passes only when **all** of these are true:
3. `web`, `api`, `ml`, and `cloudflared` are Ready. 3. `web`, `api`, `ml`, and `cloudflared` are Ready.
4. `make smoke SMOKE_SSE=1` passes and a phone on mobile data can open 4. `make smoke SMOKE_SSE=1` passes and a phone on mobile data can open
`https://pulse.noelsabia.dev`. `https://pulse.noelsabia.dev`.
5. `https://api.pulse.noelsabia.dev/api/healthz` returns HTTP 200. 5. `https://api-pulse.noelsabia.dev/api/healthz` returns HTTP 200.
Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest
dry-run, or a container that only runs on your laptop. dry-run, or a container that only runs on your laptop.
## Current external status (2026-11-14 03:50 UTC smoke) ## Current external status (2026-11-14 ~06:50 CET)
Findings from `CLUSTER-SMOKE.md`; re-check before trusting this snapshot: Re-check before trusting this snapshot:
- **Verified:** cluster access, scheduling, Service DNS/HTTP, probes, Secrets, - **Verified:** cluster access, scheduling, Service DNS/HTTP, probes, Secrets,
non-root + read-only-root + `emptyDir`, and egress to Cloudflare, non-root + read-only-root + `emptyDir`, and egress to Cloudflare,
Open-Meteo, and `git.noelsabia.dev`. Open-Meteo, and `git.noelsabia.dev`.
- **No public DNS yet:** `pulse.noelsabia.dev` and - **Tunnel active:** `pulsecast-gate1` has routes for `pulse.noelsabia.dev`
`api.pulse.noelsabia.dev` have no records. That is expected until the and `api-pulse.noelsabia.dev`; `cloudflared` is `1/1` with four HTTP/2
Cloudflare Tunnel public hostnames are created. connections, and `https://pulse.noelsabia.dev` returns 200 from hello with
- **No tunnel yet:** Cloudflare Tunnel has not been created. valid TLS. `api-pulse` returns 502 until the API is deployed.
- **Gitea pull auth is untested:** the network path works, but no - **DNS works:** public resolvers return Cloudflare IPs for
`gitea-regcred` image pull has succeeded because the read-only token does `pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`. A local
not exist yet. The hello pod in Step 2 is the first proof. `systemd-resolved` negative cache can be cleared with
`resolvectl query --cache=no NAME` followed by `resolvectl flush-caches`.
- **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only
`*.noelsabia.dev`; a two-level `api.pulse.noelsabia.dev` is not covered by
that certificate.
- **Gitea pull is proven (2026-11-14 ~06:16 CET):** `gitea-regcred` exists and
the cluster pulled `git.noelsabia.dev/noel/hello:gate1` in 120 ms; the hello
pod is `1/1 Running` and its Service returned HTTP 200. The Gitea owner is
`noel`.
- **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the - **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the
`cloudflared` manifest uses `--protocol http2` instead of relying on `cloudflared` manifest uses `--protocol http2` instead of relying on
unverified UDP. unverified UDP.
- **Numeric UIDs are required:** a non-numeric image user breaks - **Numeric UIDs are required:** a non-numeric image user breaks
`runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in `runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in
`deploy/k8s/` sets explicit numeric `runAsUser`/`runAsGroup`/`fsGroup`. `deploy/k8s/` sets explicit numeric `runAsUser`/`runAsGroup`/`fsGroup`.
- **Repo status (2026-11-14 ~05:10 UTC):** all three images now build and pass - **Repo status (2026-11-14 ~06:50 CET):** all three images build and pass
local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp` local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp`
tmpfs). There is no remaining Dockerfile blocker; the human Gitea/Tunnel tmpfs). Remaining steps are the image push, `control-auth`, deploy, and
steps are what remain. public smoke.
## What is possible now vs what is blocked ## What is possible now vs what is blocked
| Step | Who | Status | | Step | Who | Status |
|---|---|---| |---|---|---|
| Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) | | Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) |
| Gitea push login + hello-world push | human | **ready now**; registry auth still untested | | Gitea push login + hello-world push | human | **done**; hello pull and Service verified |
| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **ready now** (needs token) | | `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` + `cloudflared` **done**; `control-auth` pending |
| Cloudflare Tunnel + public hostnames/DNS | human | **blocked on human** — not created; DNS absent | | Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active, public hello 200 |
| `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally | | `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally |
| `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally | | `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally |
| `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally | | `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally |
| Full deploy + smoke + Gate 1 sign-off | human | **ready once the Gitea/Tunnel steps are done** | | Full deploy + smoke + Gate 1 sign-off | human | **ready once the tunnel/secret/image steps are done** |
`make build` and `make push` cover all three services; no Dockerfile is `make build` and `make push` cover all three services; no Dockerfile is
missing now. Do not deploy placeholder images under the real service names. missing now. Do not deploy placeholder images under the real service names.
@@ -77,14 +85,16 @@ export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarm
export NS="fch26-onemanarmy" export NS="fch26-onemanarmy"
export CTX="fch26-onemanarmy-sa-context" export CTX="fch26-onemanarmy-sa-context"
export REGISTRY="git.noelsabia.dev" export REGISTRY="git.noelsabia.dev"
export OWNER="noelsabia" export OWNER="noel"
export TAG="gate1" export TAG="gate1"
chmod 600 "$KUBECONFIG" chmod 600 "$KUBECONFIG"
``` ```
The kubeconfig is **not in this repo**. If it moved, change the path; never The kubeconfig is **not in this repo**. If it moved, change the path; never
commit it or paste its contents anywhere. commit it or paste its contents anywhere. The Gitea owner for this workspace
is `noel`; if your account differs, set `OWNER` and the image paths
consistently.
Check the local tools: Check the local tools:
@@ -142,7 +152,7 @@ Run top to bottom. Stop at the first failure and use
- [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready. - [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready.
- [ ] 11. `make smoke SMOKE_SSE=1` passes. - [ ] 11. `make smoke SMOKE_SSE=1` passes.
- [ ] 12. Phone over mobile data opens `https://pulse.noelsabia.dev`; - [ ] 12. Phone over mobile data opens `https://pulse.noelsabia.dev`;
`https://api.pulse.noelsabia.dev/api/healthz` returns 200. `https://api-pulse.noelsabia.dev/api/healthz` returns 200.
- [ ] 13. Image tags/digests and the time are recorded. - [ ] 13. Image tags/digests and the time are recorded.
## Step 0 — read-only readiness and dry-run ## Step 0 — read-only readiness and dry-run
@@ -290,29 +300,54 @@ Expected: `service/hello exposed`.
## Step 3 — Cloudflare Tunnel ## Step 3 — Cloudflare Tunnel
As of the 03:50 UTC smoke, neither public hostname has a DNS record. That is The tunnel `pulsecast-gate1` already exists. `pulse.noelsabia.dev` resolves;
expected before the tunnel route exists; do not add manual A/AAAA records. `api-pulse.noelsabia.dev` resolves after its route is saved. The old
`api.pulse.noelsabia.dev` also resolves but is **not certificate-safe**: the
Universal SSL certificate is `*.noelsabia.dev` and covers only one subdomain
level. Use `api-pulse.noelsabia.dev`, not `api.pulse.noelsabia.dev`.
```bash ```bash
getent hosts pulse.noelsabia.dev || echo 'pulse: no DNS record yet (expected)' getent hosts pulse.noelsabia.dev
getent hosts api.pulse.noelsabia.dev || echo 'api: no DNS record yet (expected)' getent hosts api-pulse.noelsabia.dev || echo 'api-pulse: no DNS record yet (expected until the route exists)'
```
If DNS looks missing locally but public DNS has it, flush the local cache:
```bash
resolvectl flush-caches
```
If `getent` still misses the name while `dig +short NAME @1.1.1.1` works, force
a fresh upstream query and flush again:
```bash
resolvectl query --cache=no api-pulse.noelsabia.dev
resolvectl flush-caches
getent hosts api-pulse.noelsabia.dev
``` ```
1. Open Cloudflare **Zero Trust → Networks → Tunnels → Create a tunnel**. 1. Open Cloudflare **Zero Trust → Networks → Tunnels → Create a tunnel**.
2. Connector: **Cloudflared**. Name: `pulsecast-gate1`. 2. Connector: **Cloudflared**. Name: `pulsecast-gate1`.
3. Save and copy the tunnel token. It is shown once. 3. Copy the tunnel token from the connector install command. It is the long
4. Open the tunnel's **Public Hostnames** tab and add: `eyJ...` string after `--token` (or after `service install`). It is
sensitive: never paste it into chat, a commit, or a screenshot. The tunnel
stays **Inactive** with 0 routes until the connector Deployment is running;
routes can be added while it is inactive.
4. Open the tunnel and click the **Routes** tab (not Overview, and not
**Add a replica**). Click **Add route** or **Add a public hostname**, choose
**Public hostname**, and add:
| Public hostname | Service | | Subdomain | Domain | Path | Service type | Service URL |
|---|---| |---|---|---|---|---|
| `pulse.noelsabia.dev` | `http://hello:8080` (temporary, Gate 1-A) | | `pulse` | `noelsabia.dev` | *(empty)* | HTTP | `hello:8080` |
| `api.pulse.noelsabia.dev` | `http://api:8080` (works after Step 10) | | `api-pulse` | `noelsabia.dev` | *(empty)* | HTTP | `api:8080` |
For the final Gate 1-B route, change `pulse.noelsabia.dev` to The first route is temporary for Gate 1-A. For the final Gate 1-B route,
`http://web:3000` after the web image is deployed. If you already have the change it to `web:3000` after the web image is deployed. If you already have
web image, you may set `web:3000` now and skip the hello route; then the the web image, you may set `web:3000` now and skip the hello route; then the
public hello check in Step 6 is not available, but the tunnel proof still public hello check in Step 6 is not available, but the tunnel proof still
happens when the real web pod is Ready. happens when the real web pod is Ready. Delete the old
`api.pulse.noelsabia.dev` route if it still exists.
5. The hostnames must be in the Cloudflare zone for `noelsabia.dev`. 5. The hostnames must be in the Cloudflare zone for `noelsabia.dev`.
Cloudflare creates the DNS records for a tunnel hostname; do **not** add Cloudflare creates the DNS records for a tunnel hostname; do **not** add
@@ -325,7 +360,7 @@ After saving both hostnames, DNS should resolve to Cloudflare:
```bash ```bash
getent hosts pulse.noelsabia.dev getent hosts pulse.noelsabia.dev
getent hosts api.pulse.noelsabia.dev getent hosts api-pulse.noelsabia.dev
``` ```
Expected: one or more Cloudflare IPv4/IPv6 addresses. If there is still no Expected: one or more Cloudflare IPv4/IPv6 addresses. If there is still no
@@ -477,9 +512,9 @@ Each service image must:
| Service | Image | Port | UID:GID | Health paths | | Service | Image | Port | UID:GID | Health paths |
|---|---|---|---|---| |---|---|---|---|---|
| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` | | web | `git.noelsabia.dev/noel/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` |
| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` | | api | `git.noelsabia.dev/noel/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` |
| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` | | ml | `git.noelsabia.dev/noel/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` |
The manifests set `runAsUser`/`runAsGroup`/`fsGroup` explicitly, so the image's The manifests set `runAsUser`/`runAsGroup`/`fsGroup` explicitly, so the image's
named user is irrelevant; the files must be readable by that numeric UID. named user is irrelevant; the files must be readable by that numeric UID.
@@ -490,7 +525,7 @@ together, and keep the value numeric.
`api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`, `api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`,
`CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and
`ML_TIMEOUT`. `ML_TIMEOUT`.
`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in `web` must have `NUXT_PUBLIC_API_BASE=https://api-pulse.noelsabia.dev` baked in
or read it at runtime from the ConfigMap; the browser must use the public API or read it at runtime from the ConfigMap; the browser must use the public API
hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`, hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`,
`PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and `PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and
@@ -615,10 +650,10 @@ Manual public checks (also run from a phone on mobile data):
```bash ```bash
curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/ curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/
curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz
curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api-pulse.noelsabia.dev/api/readyz
curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}' curl -fsS https://api-pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}'
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20 curl -sS -N --max-time 20 https://api-pulse.noelsabia.dev/api/stream | head -20
``` ```
Expected: Expected:
@@ -699,11 +734,11 @@ you need to reproduce the exact deploy.
| `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod -l app=hello` (and the real pod); `kubectl -n "$NS" get secret gitea-regcred` | The token must have `read:package`; the Secret must be in `fch26-onemanarmy`; the tag must exist in Gitea. Do not print the secret data. | | `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod -l app=hello` (and the real pod); `kubectl -n "$NS" get secret gitea-regcred` | The token must have `read:package`; the Secret must be in `fch26-onemanarmy`; the tag must exist in Gitea. Do not print the secret data. |
| `CreateContainerConfigError` | `kubectl -n "$NS" describe pod <pod>` | A referenced Secret key is missing (`control-auth`: `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`) or the `cloudflared` Secret lacks `TUNNEL_TOKEN`. Recreate the Secret. | | `CreateContainerConfigError` | `kubectl -n "$NS" describe pod <pod>` | A referenced Secret key is missing (`control-auth`: `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`) or the `cloudflared` Secret lacks `TUNNEL_TOKEN`. Recreate the Secret. |
| `container has runAsNonRoot and image will run as root` or a non-numeric-user error | `kubectl -n "$NS" get pod <pod> -o yaml`; inspect `securityContext` | Keep `runAsNonRoot: true` and set a positive numeric `runAsUser`/`runAsGroup` (SKE caveat). The manifests already do; do not "fix" this by disabling `runAsNonRoot`. | | `container has runAsNonRoot and image will run as root` or a non-numeric-user error | `kubectl -n "$NS" get pod <pod> -o yaml`; inspect `securityContext` | Keep `runAsNonRoot: true` and set a positive numeric `runAsUser`/`runAsGroup` (SKE caveat). The manifests already do; do not "fix" this by disabling `runAsNonRoot`. |
| Public hostname has no DNS record | `getent hosts pulse.noelsabia.dev` / `getent hosts api.pulse.noelsabia.dev` | Save the public hostname in the tunnel dashboard in the `noelsabia.dev` zone; remove a conflicting A/AAAA/CNAME for that exact name. Do not add manual records for a tunnel hostname. | | Public hostname has no DNS record | `getent hosts pulse.noelsabia.dev` / `getent hosts api-pulse.noelsabia.dev` | Save the public hostname in the tunnel dashboard in the `noelsabia.dev` zone; remove a conflicting A/AAAA/CNAME for that exact name. Do not add manual records for a tunnel hostname. |
| `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. | | `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. |
| `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. | | `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. |
| Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. | | Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. |
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api.pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. | | `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. |
| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. | | `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. |
| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. | | `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. |
| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. | | SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. |
+10 -10
View File
@@ -12,15 +12,15 @@ and `/api/readyz` stays 200. FastAPI being down must not fail API readiness.
```bash ```bash
# Baseline # Baseline
curl -sS https://api.pulse.noelsabia.dev/api/readyz curl -sS https://api-pulse.noelsabia.dev/api/readyz
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached' curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached'
# Stop ML # Stop ML
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=0 kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=0
# Verify fallback (allow up to 2 s for the ML timeout) # Verify fallback (allow up to 2 s for the ML timeout)
curl -sS https://api.pulse.noelsabia.dev/api/readyz curl -sS https://api-pulse.noelsabia.dev/api/readyz
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}' curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}'
# Expected: readyz 200; cached true; points still present # Expected: readyz 200; cached true; points still present
# Recover # Recover
@@ -44,7 +44,7 @@ kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployme
# Verify the outage from outside # Verify the outage from outside
curl -sS -o /dev/null -w '%{http_code}\n' https://pulse.noelsabia.dev curl -sS -o /dev/null -w '%{http_code}\n' https://pulse.noelsabia.dev
curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz curl -sS -o /dev/null -w '%{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz
# Expected: 502/503/530-style edge error, not a valid 200 # Expected: 502/503/530-style edge error, not a valid 200
# Recover # Recover
@@ -57,7 +57,7 @@ make smoke
``` ```
If the tunnel pod is Running but the URL is still 502, check the dashboard If the tunnel pod is Running but the URL is still 502, check the dashboard
routes (`pulse` -> `http://web:3000`, `api.pulse` -> `http://api:8080`) and the routes (`pulse` -> `http://web:3000`, `api-pulse` -> `http://api:8080`) and the
Secret token. A bad token keeps the pod restarting with an auth error. Rotating Secret token. A bad token keeps the pod restarting with an auth error. Rotating
the tunnel token means updating the Secret and restarting the Deployment. the tunnel token means updating the Secret and restarting the Deployment.
@@ -69,14 +69,14 @@ the default, which is documented.
```bash ```bash
# Watch from two terminals: # Watch from two terminals:
# curl -sS -N https://api.pulse.noelsabia.dev/api/stream # curl -sS -N https://api-pulse.noelsabia.dev/api/stream
# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz; sleep 1; done # while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz; sleep 1; done
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy delete pod -l app.kubernetes.io/name=api kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy delete pod -l app.kubernetes.io/name=api
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/api --timeout=180s kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/api --timeout=180s
curl -sS https://api.pulse.noelsabia.dev/api/healthz curl -sS https://api-pulse.noelsabia.dev/api/healthz
curl -sS https://api.pulse.noelsabia.dev/api/readyz curl -sS https://api-pulse.noelsabia.dev/api/readyz
``` ```
Pass: the health loop shows at most a few seconds of errors, the deployment Pass: the health loop shows at most a few seconds of errors, the deployment
+4 -4
View File
@@ -29,8 +29,8 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
1. Confirm the API is healthy and the SSE route works: 1. Confirm the API is healthy and the SSE route works:
```bash ```bash
curl -sS https://api.pulse.noelsabia.dev/api/healthz curl -sS https://api-pulse.noelsabia.dev/api/healthz
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream curl -sS -N --max-time 20 https://api-pulse.noelsabia.dev/api/stream
``` ```
You should see `retry: 3000` and a `scenario` event immediately, then a You should see `retry: 3000` and a `scenario` event immediately, then a
@@ -40,7 +40,7 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
```bash ```bash
make load make load
# or: API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js # or: API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
``` ```
`make load` writes `k6/results/summary-<timestamp>.json` (git-ignored). `make load` writes `k6/results/summary-<timestamp>.json` (git-ignored).
@@ -52,7 +52,7 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \ curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \
-H 'Content-Type: application/json' \ -H 'Content-Type: application/json' \
-d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \ -d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \
https://api.pulse.noelsabia.dev/api/scenario https://api-pulse.noelsabia.dev/api/scenario
``` ```
Repeat every 30–60 s with different scenarios if you want a distribution. Repeat every 30–60 s with different scenarios if you want a distribution.
+2 -2
View File
@@ -7,7 +7,7 @@
// make k6-build // make k6-build
// //
// Run: // Run:
// API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js // API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
// make load // make load
// //
// Recorded metrics: // Recorded metrics:
@@ -26,7 +26,7 @@ import sse from 'k6/x/sse';
import { check } from 'k6'; import { check } from 'k6';
import { Counter, Rate, Trend } from 'k6/metrics'; import { Counter, Rate, Trend } from 'k6/metrics';
const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, ''); const API_URL = (__ENV.API_URL || 'https://api-pulse.noelsabia.dev').replace(/\/+$/, '');
const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`; const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`;
const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120)); const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120));
const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20)); const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20));
+2 -2
View File
@@ -19,13 +19,13 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
NS="${NS:-fch26-onemanarmy}" NS="${NS:-fch26-onemanarmy}"
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}" CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"
REGISTRY="${REGISTRY:-git.noelsabia.dev}" REGISTRY="${REGISTRY:-git.noelsabia.dev}"
OWNER="${OWNER:-noelsabia}" OWNER="${OWNER:-noel}"
TAG="${TAG:-gate1}" TAG="${TAG:-gate1}"
CHECK_PUBLIC="${CHECK_PUBLIC:-0}" CHECK_PUBLIC="${CHECK_PUBLIC:-0}"
ALLOW_PENDING="${ALLOW_PENDING:-0}" ALLOW_PENDING="${ALLOW_PENDING:-0}"
STATIC_ONLY="${STATIC_ONLY:-0}" STATIC_ONLY="${STATIC_ONLY:-0}"
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}" WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}" API_URL="${API_URL:-https://api-pulse.noelsabia.dev}"
CURL_MAX_TIME="${CURL_MAX_TIME:-15}" CURL_MAX_TIME="${CURL_MAX_TIME:-15}"
hard=0 hard=0
+1 -1
View File
@@ -11,7 +11,7 @@
set -uo pipefail set -uo pipefail
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}" WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}" API_URL="${API_URL:-https://api-pulse.noelsabia.dev}"
ML_URL="${ML_URL:-}" ML_URL="${ML_URL:-}"
NS="${NS:-fch26-onemanarmy}" NS="${NS:-fch26-onemanarmy}"
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}" CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"