Merge pull request 'feat: infra bootstrap' (#1) from feat/infra-bootstrap into main
infra-ci / manifests (push) Failing after 4s
infra-ci / manifests (push) Failing after 4s
Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
commit
8d993c28a6
13 files changed
+115
-75
No files matched your search
@@ -10,7 +10,7 @@ KUBECONFORM ?= kubeconform
|
||||
PYTHON ?= python3
|
||||
|
||||
REGISTRY ?= git.noelsabia.dev
|
||||
OWNER ?= noelsabia
|
||||
OWNER ?= noel
|
||||
TAG ?= gate1
|
||||
WEB_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-web:$(TAG)
|
||||
API_IMAGE ?= $(REGISTRY)/$(OWNER)/pulsecast-api:$(TAG)
|
||||
@@ -27,7 +27,7 @@ API_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/api-go.git/feat-*)),$(fi
|
||||
ML_DIR ?= $(or $(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/feat-*)),$(firstword $(wildcard $(SERVICE_REPOS)/ML-py.git/main)))
|
||||
|
||||
WEB_URL ?= https://pulse.noelsabia.dev
|
||||
API_URL ?= https://api.pulse.noelsabia.dev
|
||||
API_URL ?= https://api-pulse.noelsabia.dev
|
||||
ML_URL ?=
|
||||
LOAD_API_URL ?= $(API_URL)
|
||||
SMOKE_SSE ?= 0
|
||||
|
||||
@@ -49,8 +49,13 @@ hostname, and verification steps.
|
||||
human prerequisites.
|
||||
- All three images build and pass local manifest-equivalent runtime checks
|
||||
(read-only root, `/tmp` tmpfs, numeric UID). There is no Dockerfile blocker.
|
||||
- Still human-only: Gitea read-only token + hello image pull, `gitea-regcred`,
|
||||
Cloudflare Tunnel + DNS hostnames, and the three Secrets.
|
||||
- Gitea pull is proven: `gitea-regcred` exists and the hello pod pulled from
|
||||
`git.noelsabia.dev/noel/hello:gate1`; the Service returned HTTP 200.
|
||||
- Tunnel `pulsecast-gate1` is active with `pulse` and `api-pulse` routes;
|
||||
`https://pulse.noelsabia.dev` returns 200 from hello with valid TLS. The API
|
||||
hostname is `api-pulse.noelsabia.dev` because Universal SSL only covers
|
||||
`*.noelsabia.dev`.
|
||||
- Still needed: `control-auth` Secret, image push, deploy, and public smoke.
|
||||
|
||||
## CI
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ metadata:
|
||||
app.kubernetes.io/part-of: pulsecast
|
||||
data:
|
||||
PUBLIC_WEB_URL: "https://pulse.noelsabia.dev"
|
||||
PUBLIC_API_URL: "https://api.pulse.noelsabia.dev"
|
||||
PUBLIC_API_URL: "https://api-pulse.noelsabia.dev"
|
||||
API_INTERNAL_URL: "http://api:8080"
|
||||
ML_INTERNAL_URL: "http://ml:8000"
|
||||
CORS_ALLOWED_ORIGINS: "https://pulse.noelsabia.dev,http://localhost:3000"
|
||||
@@ -28,9 +28,9 @@ data:
|
||||
PORT: "3000"
|
||||
NUXT_TELEMETRY_DISABLED: "1"
|
||||
TMPDIR: "/tmp"
|
||||
NUXT_PUBLIC_API_BASE: "https://api.pulse.noelsabia.dev"
|
||||
NUXT_PUBLIC_API_BASE: "https://api-pulse.noelsabia.dev"
|
||||
NUXT_API_INTERNAL_BASE: "http://api:8080"
|
||||
NUXT_PUBLIC_SSE_URL: "https://api.pulse.noelsabia.dev/api/stream"
|
||||
NUXT_PUBLIC_SSE_URL: "https://api-pulse.noelsabia.dev/api/stream"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
|
||||
@@ -35,7 +35,7 @@ spec:
|
||||
fsGroup: 10001
|
||||
containers:
|
||||
- name: ml
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-ml:gate1
|
||||
image: git.noelsabia.dev/noel/pulsecast-ml:gate1
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
imagePullPolicy: Always
|
||||
|
||||
@@ -36,7 +36,7 @@ spec:
|
||||
fsGroup: 65532
|
||||
containers:
|
||||
- name: api
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-api:gate1
|
||||
image: git.noelsabia.dev/noel/pulsecast-api:gate1
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
imagePullPolicy: Always
|
||||
|
||||
@@ -39,7 +39,7 @@ spec:
|
||||
containers:
|
||||
- name: web
|
||||
# Replace owner/package only if your Gitea layout differs. Keep the tag in sync with `make build`.
|
||||
image: git.noelsabia.dev/noelsabia/pulsecast-web:gate1
|
||||
image: git.noelsabia.dev/noel/pulsecast-web:gate1
|
||||
# Gate 1 reuses the `gate1` tag while iterating; Always makes a re-push visible.
|
||||
# Switch to an immutable tag + IfNotPresent at deploy freeze.
|
||||
imagePullPolicy: Always
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Cloudflare Tunnel is the only public exposure path.
|
||||
# Routes are configured in the Cloudflare Zero Trust dashboard:
|
||||
# pulse.noelsabia.dev -> http://web:3000
|
||||
# api.pulse.noelsabia.dev -> http://api:8080
|
||||
# api-pulse.noelsabia.dev -> http://api:8080
|
||||
# The token lives in the `cloudflared` Secret, created by the human; never in git.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
|
||||
+80
-45
@@ -19,48 +19,56 @@ Gate 1 passes only when **all** of these are true:
|
||||
3. `web`, `api`, `ml`, and `cloudflared` are Ready.
|
||||
4. `make smoke SMOKE_SSE=1` passes and a phone on mobile data can open
|
||||
`https://pulse.noelsabia.dev`.
|
||||
5. `https://api.pulse.noelsabia.dev/api/healthz` returns HTTP 200.
|
||||
5. `https://api-pulse.noelsabia.dev/api/healthz` returns HTTP 200.
|
||||
|
||||
Do not claim Gate 1 from a local `kubectl port-forward`, a green manifest
|
||||
dry-run, or a container that only runs on your laptop.
|
||||
|
||||
## Current external status (2026-11-14 03:50 UTC smoke)
|
||||
## Current external status (2026-11-14 ~06:50 CET)
|
||||
|
||||
Findings from `CLUSTER-SMOKE.md`; re-check before trusting this snapshot:
|
||||
Re-check before trusting this snapshot:
|
||||
|
||||
- **Verified:** cluster access, scheduling, Service DNS/HTTP, probes, Secrets,
|
||||
non-root + read-only-root + `emptyDir`, and egress to Cloudflare,
|
||||
Open-Meteo, and `git.noelsabia.dev`.
|
||||
- **No public DNS yet:** `pulse.noelsabia.dev` and
|
||||
`api.pulse.noelsabia.dev` have no records. That is expected until the
|
||||
Cloudflare Tunnel public hostnames are created.
|
||||
- **No tunnel yet:** Cloudflare Tunnel has not been created.
|
||||
- **Gitea pull auth is untested:** the network path works, but no
|
||||
`gitea-regcred` image pull has succeeded because the read-only token does
|
||||
not exist yet. The hello pod in Step 2 is the first proof.
|
||||
- **Tunnel active:** `pulsecast-gate1` has routes for `pulse.noelsabia.dev`
|
||||
and `api-pulse.noelsabia.dev`; `cloudflared` is `1/1` with four HTTP/2
|
||||
connections, and `https://pulse.noelsabia.dev` returns 200 from hello with
|
||||
valid TLS. `api-pulse` returns 502 until the API is deployed.
|
||||
- **DNS works:** public resolvers return Cloudflare IPs for
|
||||
`pulse.noelsabia.dev` and `api-pulse.noelsabia.dev`. A local
|
||||
`systemd-resolved` negative cache can be cleared with
|
||||
`resolvectl query --cache=no NAME` followed by `resolvectl flush-caches`.
|
||||
- **API hostname is `api-pulse.noelsabia.dev`:** Universal SSL covers only
|
||||
`*.noelsabia.dev`; a two-level `api.pulse.noelsabia.dev` is not covered by
|
||||
that certificate.
|
||||
- **Gitea pull is proven (2026-11-14 ~06:16 CET):** `gitea-regcred` exists and
|
||||
the cluster pulled `git.noelsabia.dev/noel/hello:gate1` in 120 ms; the hello
|
||||
pod is `1/1 Running` and its Service returned HTTP 200. The Gitea owner is
|
||||
`noel`.
|
||||
- **QUIC/UDP 7844 is untested.** TCP/443 to Cloudflare works, so the
|
||||
`cloudflared` manifest uses `--protocol http2` instead of relying on
|
||||
unverified UDP.
|
||||
- **Numeric UIDs are required:** a non-numeric image user breaks
|
||||
`runAsNonRoot` on SKE (reproduced with `curlimages/curl`). Every pod in
|
||||
`deploy/k8s/` sets explicit numeric `runAsUser`/`runAsGroup`/`fsGroup`.
|
||||
- **Repo status (2026-11-14 ~05:10 UTC):** all three images now build and pass
|
||||
- **Repo status (2026-11-14 ~06:50 CET):** all three images build and pass
|
||||
local manifest-equivalent runtime checks (numeric UID, read-only root, `/tmp`
|
||||
tmpfs). There is no remaining Dockerfile blocker; the human Gitea/Tunnel
|
||||
steps are what remain.
|
||||
tmpfs). Remaining steps are the image push, `control-auth`, deploy, and
|
||||
public smoke.
|
||||
|
||||
## What is possible now vs what is blocked
|
||||
|
||||
| Step | Who | Status |
|
||||
|---|---|---|
|
||||
| Read-only cluster/manifest check | agent or human | **ready now** (`make gate1-check`) |
|
||||
| Gitea push login + hello-world push | human | **ready now**; registry auth still untested |
|
||||
| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | **ready now** (needs token) |
|
||||
| Cloudflare Tunnel + public hostnames/DNS | human | **blocked on human** — not created; DNS absent |
|
||||
| Gitea push login + hello-world push | human | **done**; hello pull and Service verified |
|
||||
| `gitea-regcred`, `cloudflared`, `control-auth` Secrets | human | `gitea-regcred` + `cloudflared` **done**; `control-auth` pending |
|
||||
| Cloudflare Tunnel + public hostnames/DNS | human | **done**; tunnel active, public hello 200 |
|
||||
| `api` image | human | **ready** — `make build-api` + manifest-equivalent run verified locally |
|
||||
| `ml` image | human | **ready** — `make build-ml` + manifest-equivalent run verified locally |
|
||||
| `web` image | human | **ready** — `make build-web` + manifest-equivalent run verified locally |
|
||||
| Full deploy + smoke + Gate 1 sign-off | human | **ready once the Gitea/Tunnel steps are done** |
|
||||
| Full deploy + smoke + Gate 1 sign-off | human | **ready once the tunnel/secret/image steps are done** |
|
||||
|
||||
`make build` and `make push` cover all three services; no Dockerfile is
|
||||
missing now. Do not deploy placeholder images under the real service names.
|
||||
@@ -77,14 +85,16 @@ export KUBECONFIG="/home/noel/Documents/Hackathon_2026/OKF_Vault/fch26-onemanarm
|
||||
export NS="fch26-onemanarmy"
|
||||
export CTX="fch26-onemanarmy-sa-context"
|
||||
export REGISTRY="git.noelsabia.dev"
|
||||
export OWNER="noelsabia"
|
||||
export OWNER="noel"
|
||||
export TAG="gate1"
|
||||
|
||||
chmod 600 "$KUBECONFIG"
|
||||
```
|
||||
|
||||
The kubeconfig is **not in this repo**. If it moved, change the path; never
|
||||
commit it or paste its contents anywhere.
|
||||
commit it or paste its contents anywhere. The Gitea owner for this workspace
|
||||
is `noel`; if your account differs, set `OWNER` and the image paths
|
||||
consistently.
|
||||
|
||||
Check the local tools:
|
||||
|
||||
@@ -142,7 +152,7 @@ Run top to bottom. Stop at the first failure and use
|
||||
- [ ] 10. `make deploy CONFIRM=yes` completes; all four Deployments are Ready.
|
||||
- [ ] 11. `make smoke SMOKE_SSE=1` passes.
|
||||
- [ ] 12. Phone over mobile data opens `https://pulse.noelsabia.dev`;
|
||||
`https://api.pulse.noelsabia.dev/api/healthz` returns 200.
|
||||
`https://api-pulse.noelsabia.dev/api/healthz` returns 200.
|
||||
- [ ] 13. Image tags/digests and the time are recorded.
|
||||
|
||||
## Step 0 — read-only readiness and dry-run
|
||||
@@ -290,29 +300,54 @@ Expected: `service/hello exposed`.
|
||||
|
||||
## Step 3 — Cloudflare Tunnel
|
||||
|
||||
As of the 03:50 UTC smoke, neither public hostname has a DNS record. That is
|
||||
expected before the tunnel route exists; do not add manual A/AAAA records.
|
||||
The tunnel `pulsecast-gate1` already exists. `pulse.noelsabia.dev` resolves;
|
||||
`api-pulse.noelsabia.dev` resolves after its route is saved. The old
|
||||
`api.pulse.noelsabia.dev` also resolves but is **not certificate-safe**: the
|
||||
Universal SSL certificate is `*.noelsabia.dev` and covers only one subdomain
|
||||
level. Use `api-pulse.noelsabia.dev`, not `api.pulse.noelsabia.dev`.
|
||||
|
||||
```bash
|
||||
getent hosts pulse.noelsabia.dev || echo 'pulse: no DNS record yet (expected)'
|
||||
getent hosts api.pulse.noelsabia.dev || echo 'api: no DNS record yet (expected)'
|
||||
getent hosts pulse.noelsabia.dev
|
||||
getent hosts api-pulse.noelsabia.dev || echo 'api-pulse: no DNS record yet (expected until the route exists)'
|
||||
```
|
||||
|
||||
If DNS looks missing locally but public DNS has it, flush the local cache:
|
||||
|
||||
```bash
|
||||
resolvectl flush-caches
|
||||
```
|
||||
|
||||
If `getent` still misses the name while `dig +short NAME @1.1.1.1` works, force
|
||||
a fresh upstream query and flush again:
|
||||
|
||||
```bash
|
||||
resolvectl query --cache=no api-pulse.noelsabia.dev
|
||||
resolvectl flush-caches
|
||||
getent hosts api-pulse.noelsabia.dev
|
||||
```
|
||||
|
||||
1. Open Cloudflare **Zero Trust → Networks → Tunnels → Create a tunnel**.
|
||||
2. Connector: **Cloudflared**. Name: `pulsecast-gate1`.
|
||||
3. Save and copy the tunnel token. It is shown once.
|
||||
4. Open the tunnel's **Public Hostnames** tab and add:
|
||||
3. Copy the tunnel token from the connector install command. It is the long
|
||||
`eyJ...` string after `--token` (or after `service install`). It is
|
||||
sensitive: never paste it into chat, a commit, or a screenshot. The tunnel
|
||||
stays **Inactive** with 0 routes until the connector Deployment is running;
|
||||
routes can be added while it is inactive.
|
||||
4. Open the tunnel and click the **Routes** tab (not Overview, and not
|
||||
**Add a replica**). Click **Add route** or **Add a public hostname**, choose
|
||||
**Public hostname**, and add:
|
||||
|
||||
| Public hostname | Service |
|
||||
|---|---|
|
||||
| `pulse.noelsabia.dev` | `http://hello:8080` (temporary, Gate 1-A) |
|
||||
| `api.pulse.noelsabia.dev` | `http://api:8080` (works after Step 10) |
|
||||
| Subdomain | Domain | Path | Service type | Service URL |
|
||||
|---|---|---|---|---|
|
||||
| `pulse` | `noelsabia.dev` | *(empty)* | HTTP | `hello:8080` |
|
||||
| `api-pulse` | `noelsabia.dev` | *(empty)* | HTTP | `api:8080` |
|
||||
|
||||
For the final Gate 1-B route, change `pulse.noelsabia.dev` to
|
||||
`http://web:3000` after the web image is deployed. If you already have the
|
||||
web image, you may set `web:3000` now and skip the hello route; then the
|
||||
The first route is temporary for Gate 1-A. For the final Gate 1-B route,
|
||||
change it to `web:3000` after the web image is deployed. If you already have
|
||||
the web image, you may set `web:3000` now and skip the hello route; then the
|
||||
public hello check in Step 6 is not available, but the tunnel proof still
|
||||
happens when the real web pod is Ready.
|
||||
happens when the real web pod is Ready. Delete the old
|
||||
`api.pulse.noelsabia.dev` route if it still exists.
|
||||
|
||||
5. The hostnames must be in the Cloudflare zone for `noelsabia.dev`.
|
||||
Cloudflare creates the DNS records for a tunnel hostname; do **not** add
|
||||
@@ -325,7 +360,7 @@ After saving both hostnames, DNS should resolve to Cloudflare:
|
||||
|
||||
```bash
|
||||
getent hosts pulse.noelsabia.dev
|
||||
getent hosts api.pulse.noelsabia.dev
|
||||
getent hosts api-pulse.noelsabia.dev
|
||||
```
|
||||
|
||||
Expected: one or more Cloudflare IPv4/IPv6 addresses. If there is still no
|
||||
@@ -477,9 +512,9 @@ Each service image must:
|
||||
|
||||
| Service | Image | Port | UID:GID | Health paths |
|
||||
|---|---|---|---|---|
|
||||
| web | `git.noelsabia.dev/noelsabia/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` |
|
||||
| api | `git.noelsabia.dev/noelsabia/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` |
|
||||
| ml | `git.noelsabia.dev/noelsabia/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` |
|
||||
| web | `git.noelsabia.dev/noel/pulsecast-web:gate1` | 3000 | `10001:10001` | `/` |
|
||||
| api | `git.noelsabia.dev/noel/pulsecast-api:gate1` | 8080 | `65532:65532` | `/api/healthz`, `/api/readyz` |
|
||||
| ml | `git.noelsabia.dev/noel/pulsecast-ml:gate1` | 8000 | `10001:10001` | `/healthz`, `/readyz` |
|
||||
|
||||
The manifests set `runAsUser`/`runAsGroup`/`fsGroup` explicitly, so the image's
|
||||
named user is irrelevant; the files must be readable by that numeric UID.
|
||||
@@ -490,7 +525,7 @@ together, and keep the value numeric.
|
||||
`api` must read `DATA_DIR` (baked JSON), `FASTAPI_URL`, `PORT`,
|
||||
`CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`, `SSE_HEARTBEAT`, and
|
||||
`ML_TIMEOUT`.
|
||||
`web` must have `NUXT_PUBLIC_API_BASE=https://api.pulse.noelsabia.dev` baked in
|
||||
`web` must have `NUXT_PUBLIC_API_BASE=https://api-pulse.noelsabia.dev` baked in
|
||||
or read it at runtime from the ConfigMap; the browser must use the public API
|
||||
hostname. `ml` reads `PULSECAST_ROOT`, `PULSECAST_ARTIFACTS`,
|
||||
`PULSECAST_WEATHER`, `PULSECAST_MODELS`, `PULSECAST_UPLOADS`, and
|
||||
@@ -615,10 +650,10 @@ Manual public checks (also run from a phone on mobile data):
|
||||
|
||||
```bash
|
||||
curl -fsS -o /dev/null -w 'web %{http_code}\n' https://pulse.noelsabia.dev/
|
||||
curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz
|
||||
curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api.pulse.noelsabia.dev/api/readyz
|
||||
curl -fsS https://api.pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}'
|
||||
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream | head -20
|
||||
curl -fsS -o /dev/null -w 'api %{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz
|
||||
curl -fsS -o /dev/null -w 'ready %{http_code}\n' https://api-pulse.noelsabia.dev/api/readyz
|
||||
curl -fsS https://api-pulse.noelsabia.dev/api/meta | jq '{data_version,model_version}'
|
||||
curl -sS -N --max-time 20 https://api-pulse.noelsabia.dev/api/stream | head -20
|
||||
```
|
||||
|
||||
Expected:
|
||||
@@ -699,11 +734,11 @@ you need to reproduce the exact deploy.
|
||||
| `ImagePullBackOff` / `ErrImagePull` | `kubectl -n "$NS" describe pod -l app=hello` (and the real pod); `kubectl -n "$NS" get secret gitea-regcred` | The token must have `read:package`; the Secret must be in `fch26-onemanarmy`; the tag must exist in Gitea. Do not print the secret data. |
|
||||
| `CreateContainerConfigError` | `kubectl -n "$NS" describe pod <pod>` | A referenced Secret key is missing (`control-auth`: `CONTROL_USER`, `CONTROL_PASS`, `CONTROL_TOKEN`) or the `cloudflared` Secret lacks `TUNNEL_TOKEN`. Recreate the Secret. |
|
||||
| `container has runAsNonRoot and image will run as root` or a non-numeric-user error | `kubectl -n "$NS" get pod <pod> -o yaml`; inspect `securityContext` | Keep `runAsNonRoot: true` and set a positive numeric `runAsUser`/`runAsGroup` (SKE caveat). The manifests already do; do not "fix" this by disabling `runAsNonRoot`. |
|
||||
| Public hostname has no DNS record | `getent hosts pulse.noelsabia.dev` / `getent hosts api.pulse.noelsabia.dev` | Save the public hostname in the tunnel dashboard in the `noelsabia.dev` zone; remove a conflicting A/AAAA/CNAME for that exact name. Do not add manual records for a tunnel hostname. |
|
||||
| Public hostname has no DNS record | `getent hosts pulse.noelsabia.dev` / `getent hosts api-pulse.noelsabia.dev` | Save the public hostname in the tunnel dashboard in the `noelsabia.dev` zone; remove a conflicting A/AAAA/CNAME for that exact name. Do not add manual records for a tunnel hostname. |
|
||||
| `cloudflared` → `CrashLoopBackOff` | `kubectl -n "$NS" logs deployment/cloudflared --previous` | Invalid/expired tunnel token, or the tunnel was deleted in Cloudflare. Recreate the Secret and restart. |
|
||||
| `cloudflared` logs QUIC/timeout errors after you changed the protocol | `kubectl -n "$NS" logs deployment/cloudflared --tail=50` | UDP/7844 is unverified on SKE. Put `--protocol http2` back (the committed manifest already sets it); TCP/443 is confirmed. |
|
||||
| Public `502` / `1033` / `530` | `kubectl -n "$NS" get endpoints web api hello`; cloudflared logs | No ready endpoints, wrong Service port, or the hostname route points at the wrong service. Fix the dashboard route or the Service. Never add an Ingress. |
|
||||
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api.pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. |
|
||||
| `api` pod not ready | `kubectl -n "$NS" logs deployment/api`; `curl -fsS https://api-pulse.noelsabia.dev/api/readyz` | Baked JSON missing from the image or `DATA_DIR` wrong. Rebuild the API image with the artifacts. The distroless API image has no shell, so do not rely on `kubectl exec ... ls`. |
|
||||
| `ml` pod stuck in startup | `kubectl -n "$NS" logs deployment/ml` | Model load exceeds the 120 s startup budget or `MODEL_PATH` is wrong. Fix the image/path. |
|
||||
| `web` 500 | `kubectl -n "$NS" logs deployment/web` | Missing runtime API base or SSR error. Rebuild with `NUXT_PUBLIC_API_BASE` or fix the ConfigMap. |
|
||||
| SSE events arrive only after a delay | `curl -N` direct to the pod vs through Cloudflare | Cloudflare buffering. Keep the 15 s heartbeat and the web polling fallback (`/api/scenario` every 5 s). Do not remove the heartbeat. |
|
||||
|
||||
+10
-10
@@ -12,15 +12,15 @@ and `/api/readyz` stays 200. FastAPI being down must not fail API readiness.
|
||||
|
||||
```bash
|
||||
# Baseline
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached'
|
||||
curl -sS https://api-pulse.noelsabia.dev/api/readyz
|
||||
curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '.cached'
|
||||
|
||||
# Stop ML
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployment/ml --replicas=0
|
||||
|
||||
# Verify fallback (allow up to 2 s for the ML timeout)
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||
curl -sS 'https://api.pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}'
|
||||
curl -sS https://api-pulse.noelsabia.dev/api/readyz
|
||||
curl -sS 'https://api-pulse.noelsabia.dev/api/forecast?sensor=hn-stadtgalerie-01&horizon=7d&scenario=base' | jq '{cached, points: (.points|length)}'
|
||||
# Expected: readyz 200; cached true; points still present
|
||||
|
||||
# Recover
|
||||
@@ -44,7 +44,7 @@ kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy scale deployme
|
||||
|
||||
# Verify the outage from outside
|
||||
curl -sS -o /dev/null -w '%{http_code}\n' https://pulse.noelsabia.dev
|
||||
curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz
|
||||
curl -sS -o /dev/null -w '%{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz
|
||||
# Expected: 502/503/530-style edge error, not a valid 200
|
||||
|
||||
# Recover
|
||||
@@ -57,7 +57,7 @@ make smoke
|
||||
```
|
||||
|
||||
If the tunnel pod is Running but the URL is still 502, check the dashboard
|
||||
routes (`pulse` -> `http://web:3000`, `api.pulse` -> `http://api:8080`) and the
|
||||
routes (`pulse` -> `http://web:3000`, `api-pulse` -> `http://api:8080`) and the
|
||||
Secret token. A bad token keeps the pod restarting with an auth error. Rotating
|
||||
the tunnel token means updating the Secret and restarting the Deployment.
|
||||
|
||||
@@ -69,14 +69,14 @@ the default, which is documented.
|
||||
|
||||
```bash
|
||||
# Watch from two terminals:
|
||||
# curl -sS -N https://api.pulse.noelsabia.dev/api/stream
|
||||
# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api.pulse.noelsabia.dev/api/healthz; sleep 1; done
|
||||
# curl -sS -N https://api-pulse.noelsabia.dev/api/stream
|
||||
# while true; do date +%T; curl -sS -o /dev/null -w '%{http_code}\n' https://api-pulse.noelsabia.dev/api/healthz; sleep 1; done
|
||||
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy delete pod -l app.kubernetes.io/name=api
|
||||
|
||||
kubectl --context fch26-onemanarmy-sa-context -n fch26-onemanarmy rollout status deployment/api --timeout=180s
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/healthz
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/readyz
|
||||
curl -sS https://api-pulse.noelsabia.dev/api/healthz
|
||||
curl -sS https://api-pulse.noelsabia.dev/api/readyz
|
||||
```
|
||||
|
||||
Pass: the health loop shows at most a few seconds of errors, the deployment
|
||||
|
||||
+4
-4
@@ -29,8 +29,8 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
|
||||
1. Confirm the API is healthy and the SSE route works:
|
||||
|
||||
```bash
|
||||
curl -sS https://api.pulse.noelsabia.dev/api/healthz
|
||||
curl -sS -N --max-time 20 https://api.pulse.noelsabia.dev/api/stream
|
||||
curl -sS https://api-pulse.noelsabia.dev/api/healthz
|
||||
curl -sS -N --max-time 20 https://api-pulse.noelsabia.dev/api/stream
|
||||
```
|
||||
|
||||
You should see `retry: 3000` and a `scenario` event immediately, then a
|
||||
@@ -40,7 +40,7 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
|
||||
|
||||
```bash
|
||||
make load
|
||||
# or: API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
|
||||
# or: API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
|
||||
```
|
||||
|
||||
`make load` writes `k6/results/summary-<timestamp>.json` (git-ignored).
|
||||
@@ -52,7 +52,7 @@ A stock `grafana/k6` image is **not** a fallback: it lacks the SSE extension.
|
||||
curl -u "$CONTROL_USER:$CONTROL_PASS" -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0}}' \
|
||||
https://api.pulse.noelsabia.dev/api/scenario
|
||||
https://api-pulse.noelsabia.dev/api/scenario
|
||||
```
|
||||
|
||||
Repeat every 30–60 s with different scenarios if you want a distribution.
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
// make k6-build
|
||||
//
|
||||
// Run:
|
||||
// API_URL=https://api.pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
|
||||
// API_URL=https://api-pulse.noelsabia.dev ./k6/k6-sse run k6/sse.js
|
||||
// make load
|
||||
//
|
||||
// Recorded metrics:
|
||||
@@ -26,7 +26,7 @@ import sse from 'k6/x/sse';
|
||||
import { check } from 'k6';
|
||||
import { Counter, Rate, Trend } from 'k6/metrics';
|
||||
|
||||
const API_URL = (__ENV.API_URL || 'https://api.pulse.noelsabia.dev').replace(/\/+$/, '');
|
||||
const API_URL = (__ENV.API_URL || 'https://api-pulse.noelsabia.dev').replace(/\/+$/, '');
|
||||
const STREAM_URL = __ENV.SSE_URL || `${API_URL}/api/stream`;
|
||||
const VUS = Math.max(1, Number(__ENV.SSE_VUS || 120));
|
||||
const HOLD_PINGS = Math.max(1, Number(__ENV.SSE_HOLD_PINGS || 20));
|
||||
|
||||
@@ -19,13 +19,13 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
NS="${NS:-fch26-onemanarmy}"
|
||||
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"
|
||||
REGISTRY="${REGISTRY:-git.noelsabia.dev}"
|
||||
OWNER="${OWNER:-noelsabia}"
|
||||
OWNER="${OWNER:-noel}"
|
||||
TAG="${TAG:-gate1}"
|
||||
CHECK_PUBLIC="${CHECK_PUBLIC:-0}"
|
||||
ALLOW_PENDING="${ALLOW_PENDING:-0}"
|
||||
STATIC_ONLY="${STATIC_ONLY:-0}"
|
||||
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
|
||||
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}"
|
||||
API_URL="${API_URL:-https://api-pulse.noelsabia.dev}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-15}"
|
||||
|
||||
hard=0
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
set -uo pipefail
|
||||
|
||||
WEB_URL="${WEB_URL:-https://pulse.noelsabia.dev}"
|
||||
API_URL="${API_URL:-https://api.pulse.noelsabia.dev}"
|
||||
API_URL="${API_URL:-https://api-pulse.noelsabia.dev}"
|
||||
ML_URL="${ML_URL:-}"
|
||||
NS="${NS:-fch26-onemanarmy}"
|
||||
CONTEXT="${CONTEXT:-fch26-onemanarmy-sa-context}"
|
||||
|
||||
Reference in new issue
Block a user