feat: add /bootstrap snapshot, numeric UID 10001, CI gate
GET /bootstrap returns the 8 sensors, hourly series for all of them, context, meta, and versions in the CONTRACT.md shape. Snapshot artifacts are projected to contract fields once at boot; incomplete artifacts return 503. Adds tests for shape, sensor count, hourly alignment, and versions. Dockerfile now creates UID/GID 10001, uses COPY --chown=10001:10001, and runs as USER 10001:10001. Adds make check and a minimal Gitea Actions workflow that runs the same local gate: uv sync --frozen, compileall, pytest.
This commit is contained in:
1 parent
7ec4fc8bb3
commit
8656d9bc8a
6 files changed
+210
-12
No files matched your search
@@ -0,0 +1,31 @@
|
|||||||
|
name: ci
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
- "feat/**"
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Check out
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
run: python -m pip install "uv==0.10.4"
|
||||||
|
|
||||||
|
# Same gate as local: uv sync --frozen, compileall src, pytest.
|
||||||
|
# No secrets, no image push, no cluster access.
|
||||||
|
- name: Check
|
||||||
|
run: make check
|
||||||
+9
-8
@@ -24,17 +24,18 @@ ENV PYTHONUNBUFFERED=1 \
|
|||||||
PULSECAST_UPLOADS=/tmp/pulsecast/uploads \
|
PULSECAST_UPLOADS=/tmp/pulsecast/uploads \
|
||||||
PATH=/app/.venv/bin:$PATH
|
PATH=/app/.venv/bin:$PATH
|
||||||
|
|
||||||
RUN groupadd --system app \
|
RUN groupadd --gid 10001 app \
|
||||||
&& useradd --system --gid app --home-dir /app --shell /usr/sbin/nologin app \
|
&& useradd --uid 10001 --gid 10001 --home-dir /app --no-create-home \
|
||||||
&& mkdir -p /tmp/pulsecast/models /tmp/pulsecast/uploads \
|
--shell /usr/sbin/nologin app \
|
||||||
&& chown -R app:app /tmp/pulsecast
|
&& install -d -o 10001 -g 10001 -m 0755 \
|
||||||
|
/tmp/pulsecast /tmp/pulsecast/models /tmp/pulsecast/uploads
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --chown=app:app --from=builder /app/.venv /app/.venv
|
COPY --chown=10001:10001 --from=builder /app/.venv /app/.venv
|
||||||
COPY --chown=app:app artifacts ./artifacts
|
COPY --chown=10001:10001 artifacts ./artifacts
|
||||||
COPY --chown=app:app main.py README.md model_card.md ./
|
COPY --chown=10001:10001 main.py README.md model_card.md ./
|
||||||
|
|
||||||
USER app
|
USER 10001:10001
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
|
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
UV ?= uv
|
UV ?= uv
|
||||||
|
|
||||||
.PHONY: help data train test serve docker clean
|
.PHONY: help data train test check serve docker clean
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@printf '%s\n' \
|
@printf '%s\n' \
|
||||||
'make data parse/clean the raw CSV, join weather + holidays, export JSON' \
|
'make data parse/clean the raw CSV, join weather + holidays, export JSON' \
|
||||||
'make train fit the 28-day holdout model, export metrics/forecasts/model' \
|
'make train fit the 28-day holdout model, export metrics/forecasts/model' \
|
||||||
'make test run pytest' \
|
'make test run pytest' \
|
||||||
|
'make check CI gate: uv sync --frozen, compileall, pytest' \
|
||||||
'make serve run the FastAPI service on :8000' \
|
'make serve run the FastAPI service on :8000' \
|
||||||
'make docker build the multi-stage image (pulsecast-ml:local)' \
|
'make docker build the multi-stage image (pulsecast-ml:local)' \
|
||||||
'make clean remove caches and generated artifacts (keeps data/raw)'
|
'make clean remove caches and generated artifacts (keeps data/raw)'
|
||||||
@@ -20,6 +21,11 @@ train:
|
|||||||
test:
|
test:
|
||||||
$(UV) run pytest
|
$(UV) run pytest
|
||||||
|
|
||||||
|
check:
|
||||||
|
$(UV) sync --frozen
|
||||||
|
$(UV) run python -m compileall src
|
||||||
|
$(UV) run pytest -q
|
||||||
|
|
||||||
serve:
|
serve:
|
||||||
$(UV) run uvicorn pulsecast.server:app --host 0.0.0.0 --port 8000
|
$(UV) run uvicorn pulsecast.server:app --host 0.0.0.0 --port 8000
|
||||||
|
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ uv sync
|
|||||||
make data # needs the raw CSV, first run also fetches Open-Meteo weather
|
make data # needs the raw CSV, first run also fetches Open-Meteo weather
|
||||||
make train # writes artifacts/model.joblib, metrics.json, forecasts.json
|
make train # writes artifacts/model.joblib, metrics.json, forecasts.json
|
||||||
make test
|
make test
|
||||||
|
make check # CI gate: uv sync --frozen, compileall, pytest
|
||||||
make serve # http://localhost:8000
|
make serve # http://localhost:8000
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -84,6 +85,7 @@ Shapes follow `CONTRACT.md`. The service is internal; Go exposes it publicly.
|
|||||||
| `GET` | `/healthz` | `{"ok":true}` |
|
| `GET` | `/healthz` | `{"ok":true}` |
|
||||||
| `GET` | `/readyz` | `{"ok":true,"model":true}`, 503 if no model |
|
| `GET` | `/readyz` | `{"ok":true,"model":true}`, 503 if no model |
|
||||||
| `GET` | `/meta` | Versions, coverage, metrics, features, holiday effect |
|
| `GET` | `/meta` | Versions, coverage, metrics, features, holiday effect |
|
||||||
|
| `GET` | `/bootstrap` | Snapshot for Go: sensors, hourly series, context, meta |
|
||||||
| `POST` | `/predict` | `{sensor,horizon,scenario,weather?,is_holiday?}` → `points` |
|
| `POST` | `/predict` | `{sensor,horizon,scenario,weather?,is_holiday?}` → `points` |
|
||||||
| `POST` | `/simulate` | Scenario forecast for all or selected sensors |
|
| `POST` | `/simulate` | Scenario forecast for all or selected sensors |
|
||||||
| `POST` | `/train` | Control token; multipart CSV or JSON path; 202 job |
|
| `POST` | `/train` | Control token; multipart CSV or JSON path; 202 job |
|
||||||
@@ -123,8 +125,8 @@ make docker
|
|||||||
docker run --rm -p 8000:8000 -e CONTROL_TOKEN=change-me pulsecast-ml:local
|
docker run --rm -p 8000:8000 -e CONTROL_TOKEN=change-me pulsecast-ml:local
|
||||||
```
|
```
|
||||||
|
|
||||||
Multi-stage `python:3.12-slim`, non-root user, model and artifacts baked in,
|
Multi-stage `python:3.12-slim`, numeric non-root user `10001:10001`, model and
|
||||||
writable paths only under `/tmp`.
|
artifacts baked in, writable paths only under `/tmp`.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
|
|||||||
+70
-1
@@ -1,4 +1,4 @@
|
|||||||
"""FastAPI ML service: /healthz, /readyz, /meta, /predict, /simulate, /train.
|
"""FastAPI ML service: /healthz, /readyz, /meta, /bootstrap, /predict, /simulate, /train.
|
||||||
|
|
||||||
Shapes follow CONTRACT.md. The service is internal: only Go exposes it publicly,
|
Shapes follow CONTRACT.md. The service is internal: only Go exposes it publicly,
|
||||||
and the training routes require a bearer control token.
|
and the training routes require a bearer control token.
|
||||||
@@ -36,6 +36,7 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
MAX_UPLOAD_BYTES = 60 * 1024 * 1024
|
MAX_UPLOAD_BYTES = 60 * 1024 * 1024
|
||||||
ACTIVE_STATUSES = {"queued", "running"}
|
ACTIVE_STATUSES = {"queued", "running"}
|
||||||
|
SENSOR_FIELDS = ("id", "name", "lat", "lon", "mean", "p95", "battery")
|
||||||
|
|
||||||
|
|
||||||
def _utc_now() -> str:
|
def _utc_now() -> str:
|
||||||
@@ -48,6 +49,9 @@ class Runtime:
|
|||||||
metrics: dict | None = None
|
metrics: dict | None = None
|
||||||
weather: pd.DataFrame | None = None
|
weather: pd.DataFrame | None = None
|
||||||
weather_meta: dict | None = None
|
weather_meta: dict | None = None
|
||||||
|
sensors: list[dict] | None = None
|
||||||
|
series: dict[str, list[dict]] | None = None
|
||||||
|
context: dict | None = None
|
||||||
small_bundle: dict | None = None
|
small_bundle: dict | None = None
|
||||||
small_metrics: dict | None = None
|
small_metrics: dict | None = None
|
||||||
model_source: str = "baked"
|
model_source: str = "baked"
|
||||||
@@ -75,6 +79,38 @@ def _load_runtime(runtime: Runtime) -> None:
|
|||||||
json.loads(meta_path.read_text()) if meta_path.exists() else None
|
json.loads(meta_path.read_text()) if meta_path.exists() else None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Baked snapshot artifacts served by GET /bootstrap. Go caches this once at
|
||||||
|
# startup, so loading them at boot keeps the endpoint a pure in-memory read.
|
||||||
|
# Project to the contract fields; artifacts carry extra audit fields
|
||||||
|
# (last_seen) that Go does not need.
|
||||||
|
sensors_path = artifacts / "sensors.json"
|
||||||
|
if sensors_path.exists():
|
||||||
|
sensors = json.loads(sensors_path.read_text())
|
||||||
|
runtime.sensors = [
|
||||||
|
{key: sensor.get(key) for key in SENSOR_FIELDS} for sensor in sensors
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
log.warning("snapshot artifact missing: %s", sensors_path)
|
||||||
|
|
||||||
|
series_path = artifacts / "series.json"
|
||||||
|
if series_path.exists():
|
||||||
|
series = json.loads(series_path.read_text()).get("series") or {}
|
||||||
|
runtime.series = {
|
||||||
|
sensor: [
|
||||||
|
{"t": point["t"], "v": point["v"]}
|
||||||
|
for point in series.get(sensor, [])
|
||||||
|
]
|
||||||
|
for sensor in config.USABLE_SENSORS
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
log.warning("snapshot artifact missing: %s", series_path)
|
||||||
|
|
||||||
|
context_path = artifacts / "context.json"
|
||||||
|
if context_path.exists():
|
||||||
|
runtime.context = json.loads(context_path.read_text())
|
||||||
|
else:
|
||||||
|
log.warning("snapshot artifact missing: %s", context_path)
|
||||||
|
|
||||||
small_path = artifacts / "small_model.joblib"
|
small_path = artifacts / "small_model.joblib"
|
||||||
small_metrics_path = artifacts / "small_metrics.json"
|
small_metrics_path = artifacts / "small_metrics.json"
|
||||||
if small_path.exists():
|
if small_path.exists():
|
||||||
@@ -468,6 +504,39 @@ def create_app() -> FastAPI:
|
|||||||
}
|
}
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
@app.get("/bootstrap")
|
||||||
|
def bootstrap() -> dict:
|
||||||
|
"""Snapshot Go caches at startup: sensors, hourly series, context, meta."""
|
||||||
|
state = runtime()
|
||||||
|
if state.bundle is None or state.metrics is None:
|
||||||
|
raise HTTPException(503, "model is not loaded")
|
||||||
|
if state.sensors is None or state.series is None or state.context is None:
|
||||||
|
raise HTTPException(503, "snapshot artifacts are not loaded")
|
||||||
|
if len(state.sensors) != len(config.USABLE_SENSORS) or not all(
|
||||||
|
state.series.get(sensor) for sensor in config.USABLE_SENSORS
|
||||||
|
):
|
||||||
|
raise HTTPException(503, "snapshot is incomplete")
|
||||||
|
|
||||||
|
metrics = state.metrics
|
||||||
|
context = state.context
|
||||||
|
return {
|
||||||
|
"data_version": metrics.get("data_version"),
|
||||||
|
"model_version": metrics.get("model_version"),
|
||||||
|
"context_version": metrics.get("context_version"),
|
||||||
|
"sensors": state.sensors,
|
||||||
|
"series": state.series,
|
||||||
|
"context": {
|
||||||
|
"weather": context.get("weather"),
|
||||||
|
"holidays": context.get("holidays"),
|
||||||
|
"updated_at": context.get("updated_at"),
|
||||||
|
},
|
||||||
|
"meta": {
|
||||||
|
"coverage": metrics.get("coverage"),
|
||||||
|
"metrics": metrics.get("metrics"),
|
||||||
|
"limits": metrics.get("limits", []),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
@app.post("/predict")
|
@app.post("/predict")
|
||||||
def predict_endpoint(body: PredictRequest) -> dict:
|
def predict_endpoint(body: PredictRequest) -> dict:
|
||||||
state = runtime()
|
state = runtime()
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from pulsecast import config
|
||||||
|
|
||||||
|
BOOTSTRAP_KEYS = {
|
||||||
|
"data_version",
|
||||||
|
"model_version",
|
||||||
|
"context_version",
|
||||||
|
"sensors",
|
||||||
|
"series",
|
||||||
|
"context",
|
||||||
|
"meta",
|
||||||
|
}
|
||||||
|
|
||||||
|
SENSOR_KEYS = {"id", "name", "lat", "lon", "mean", "p95", "battery"}
|
||||||
|
|
||||||
|
|
||||||
|
def _timestamp(value: str) -> datetime:
|
||||||
|
assert value.endswith("Z")
|
||||||
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_bootstrap_shape(client):
|
||||||
|
response = client.get("/bootstrap")
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert response.headers["content-type"].startswith("application/json")
|
||||||
|
payload = response.json()
|
||||||
|
|
||||||
|
assert set(payload) == BOOTSTRAP_KEYS
|
||||||
|
_timestamp(payload["data_version"])
|
||||||
|
_timestamp(payload["context_version"])
|
||||||
|
assert payload["model_version"].startswith("histgb-")
|
||||||
|
|
||||||
|
sensors = payload["sensors"]
|
||||||
|
assert len(sensors) == 8
|
||||||
|
assert {sensor["id"] for sensor in sensors} == set(config.USABLE_SENSORS)
|
||||||
|
assert [sensor["id"] for sensor in sensors] == sorted(sensor["id"] for sensor in sensors)
|
||||||
|
for sensor in sensors:
|
||||||
|
assert set(sensor) == SENSOR_KEYS
|
||||||
|
assert sensor["p95"] >= sensor["mean"]
|
||||||
|
assert sensor["battery"] is None or 0 <= sensor["battery"] <= 100
|
||||||
|
|
||||||
|
|
||||||
|
def test_bootstrap_series_is_hourly_for_all_eight_sensors(client):
|
||||||
|
series = client.get("/bootstrap").json()["series"]
|
||||||
|
assert set(series) == set(config.USABLE_SENSORS)
|
||||||
|
|
||||||
|
total_points = 0
|
||||||
|
for sensor, points in series.items():
|
||||||
|
assert len(points) > 1000, sensor
|
||||||
|
total_points += len(points)
|
||||||
|
previous = None
|
||||||
|
for point in points:
|
||||||
|
assert set(point) == {"t", "v"}
|
||||||
|
timestamp = _timestamp(point["t"])
|
||||||
|
assert timestamp.minute == 0 and timestamp.second == 0
|
||||||
|
if previous is not None:
|
||||||
|
assert timestamp > previous
|
||||||
|
previous = timestamp
|
||||||
|
assert point["v"] >= 0
|
||||||
|
assert total_points > 20_000
|
||||||
|
|
||||||
|
|
||||||
|
def test_bootstrap_context_and_meta(client):
|
||||||
|
payload = client.get("/bootstrap").json()
|
||||||
|
|
||||||
|
context = payload["context"]
|
||||||
|
assert set(context) == {"weather", "holidays", "updated_at"}
|
||||||
|
assert set(context["weather"]) == {"now", "forecast"}
|
||||||
|
assert len(context["weather"]["forecast"]) == 168
|
||||||
|
assert context["holidays"]
|
||||||
|
assert {"date", "name"} == set(context["holidays"][0])
|
||||||
|
_timestamp(context["updated_at"])
|
||||||
|
|
||||||
|
meta = payload["meta"]
|
||||||
|
assert set(meta) == {"coverage", "metrics", "limits"}
|
||||||
|
assert meta["coverage"]["sensors"] == 8
|
||||||
|
assert meta["coverage"]["cadence_min"] == 15
|
||||||
|
assert meta["metrics"]["mae"] < meta["metrics"]["baseline_mae"]
|
||||||
|
assert len(meta["limits"]) >= 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_bootstrap_versions_match_meta(client):
|
||||||
|
payload = client.get("/bootstrap").json()
|
||||||
|
meta = client.get("/meta").json()
|
||||||
|
for key in ("data_version", "model_version", "context_version"):
|
||||||
|
assert payload[key] == meta[key]
|
||||||
Reference in new issue
Block a user