diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..fc7cbe3 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,31 @@ +name: ci + +on: + push: + branches: + - main + - "feat/**" + pull_request: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + run: python -m pip install "uv==0.10.4" + + # Same gate as local: uv sync --frozen, compileall src, pytest. + # No secrets, no image push, no cluster access. + - name: Check + run: make check diff --git a/Dockerfile b/Dockerfile index 8be7296..113eaea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,17 +24,18 @@ ENV PYTHONUNBUFFERED=1 \ PULSECAST_UPLOADS=/tmp/pulsecast/uploads \ PATH=/app/.venv/bin:$PATH -RUN groupadd --system app \ - && useradd --system --gid app --home-dir /app --shell /usr/sbin/nologin app \ - && mkdir -p /tmp/pulsecast/models /tmp/pulsecast/uploads \ - && chown -R app:app /tmp/pulsecast +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid 10001 --home-dir /app --no-create-home \ + --shell /usr/sbin/nologin app \ + && install -d -o 10001 -g 10001 -m 0755 \ + /tmp/pulsecast /tmp/pulsecast/models /tmp/pulsecast/uploads WORKDIR /app -COPY --chown=app:app --from=builder /app/.venv /app/.venv -COPY --chown=app:app artifacts ./artifacts -COPY --chown=app:app main.py README.md model_card.md ./ +COPY --chown=10001:10001 --from=builder /app/.venv /app/.venv +COPY --chown=10001:10001 artifacts ./artifacts +COPY --chown=10001:10001 main.py README.md model_card.md ./ -USER app +USER 10001:10001 EXPOSE 8000 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ diff --git a/Makefile b/Makefile index e7dfa0d..9cac739 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,13 @@ UV ?= uv -.PHONY: help data train test serve docker clean +.PHONY: help data train test check serve docker clean help: @printf '%s\n' \ 'make data parse/clean the raw CSV, join weather + holidays, export JSON' \ 'make train fit the 28-day holdout model, export metrics/forecasts/model' \ 'make test run pytest' \ + 'make check CI gate: uv sync --frozen, compileall, pytest' \ 'make serve run the FastAPI service on :8000' \ 'make docker build the multi-stage image (pulsecast-ml:local)' \ 'make clean remove caches and generated artifacts (keeps data/raw)' @@ -20,6 +21,11 @@ train: test: $(UV) run pytest +check: + $(UV) sync --frozen + $(UV) run python -m compileall src + $(UV) run pytest -q + serve: $(UV) run uvicorn pulsecast.server:app --host 0.0.0.0 --port 8000 diff --git a/README.md b/README.md index 6c91860..d574c56 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ uv sync make data # needs the raw CSV, first run also fetches Open-Meteo weather make train # writes artifacts/model.joblib, metrics.json, forecasts.json make test +make check # CI gate: uv sync --frozen, compileall, pytest make serve # http://localhost:8000 ``` @@ -84,6 +85,7 @@ Shapes follow `CONTRACT.md`. The service is internal; Go exposes it publicly. | `GET` | `/healthz` | `{"ok":true}` | | `GET` | `/readyz` | `{"ok":true,"model":true}`, 503 if no model | | `GET` | `/meta` | Versions, coverage, metrics, features, holiday effect | +| `GET` | `/bootstrap` | Snapshot for Go: sensors, hourly series, context, meta | | `POST` | `/predict` | `{sensor,horizon,scenario,weather?,is_holiday?}` → `points` | | `POST` | `/simulate` | Scenario forecast for all or selected sensors | | `POST` | `/train` | Control token; multipart CSV or JSON path; 202 job | @@ -123,8 +125,8 @@ make docker docker run --rm -p 8000:8000 -e CONTROL_TOKEN=change-me pulsecast-ml:local ``` -Multi-stage `python:3.12-slim`, non-root user, model and artifacts baked in, -writable paths only under `/tmp`. +Multi-stage `python:3.12-slim`, numeric non-root user `10001:10001`, model and +artifacts baked in, writable paths only under `/tmp`. ## Configuration diff --git a/src/pulsecast/server.py b/src/pulsecast/server.py index 4e1e358..e67e4ac 100644 --- a/src/pulsecast/server.py +++ b/src/pulsecast/server.py @@ -1,4 +1,4 @@ -"""FastAPI ML service: /healthz, /readyz, /meta, /predict, /simulate, /train. +"""FastAPI ML service: /healthz, /readyz, /meta, /bootstrap, /predict, /simulate, /train. Shapes follow CONTRACT.md. The service is internal: only Go exposes it publicly, and the training routes require a bearer control token. @@ -36,6 +36,7 @@ log = logging.getLogger(__name__) MAX_UPLOAD_BYTES = 60 * 1024 * 1024 ACTIVE_STATUSES = {"queued", "running"} +SENSOR_FIELDS = ("id", "name", "lat", "lon", "mean", "p95", "battery") def _utc_now() -> str: @@ -48,6 +49,9 @@ class Runtime: metrics: dict | None = None weather: pd.DataFrame | None = None weather_meta: dict | None = None + sensors: list[dict] | None = None + series: dict[str, list[dict]] | None = None + context: dict | None = None small_bundle: dict | None = None small_metrics: dict | None = None model_source: str = "baked" @@ -75,6 +79,38 @@ def _load_runtime(runtime: Runtime) -> None: json.loads(meta_path.read_text()) if meta_path.exists() else None ) + # Baked snapshot artifacts served by GET /bootstrap. Go caches this once at + # startup, so loading them at boot keeps the endpoint a pure in-memory read. + # Project to the contract fields; artifacts carry extra audit fields + # (last_seen) that Go does not need. + sensors_path = artifacts / "sensors.json" + if sensors_path.exists(): + sensors = json.loads(sensors_path.read_text()) + runtime.sensors = [ + {key: sensor.get(key) for key in SENSOR_FIELDS} for sensor in sensors + ] + else: + log.warning("snapshot artifact missing: %s", sensors_path) + + series_path = artifacts / "series.json" + if series_path.exists(): + series = json.loads(series_path.read_text()).get("series") or {} + runtime.series = { + sensor: [ + {"t": point["t"], "v": point["v"]} + for point in series.get(sensor, []) + ] + for sensor in config.USABLE_SENSORS + } + else: + log.warning("snapshot artifact missing: %s", series_path) + + context_path = artifacts / "context.json" + if context_path.exists(): + runtime.context = json.loads(context_path.read_text()) + else: + log.warning("snapshot artifact missing: %s", context_path) + small_path = artifacts / "small_model.joblib" small_metrics_path = artifacts / "small_metrics.json" if small_path.exists(): @@ -468,6 +504,39 @@ def create_app() -> FastAPI: } return payload + @app.get("/bootstrap") + def bootstrap() -> dict: + """Snapshot Go caches at startup: sensors, hourly series, context, meta.""" + state = runtime() + if state.bundle is None or state.metrics is None: + raise HTTPException(503, "model is not loaded") + if state.sensors is None or state.series is None or state.context is None: + raise HTTPException(503, "snapshot artifacts are not loaded") + if len(state.sensors) != len(config.USABLE_SENSORS) or not all( + state.series.get(sensor) for sensor in config.USABLE_SENSORS + ): + raise HTTPException(503, "snapshot is incomplete") + + metrics = state.metrics + context = state.context + return { + "data_version": metrics.get("data_version"), + "model_version": metrics.get("model_version"), + "context_version": metrics.get("context_version"), + "sensors": state.sensors, + "series": state.series, + "context": { + "weather": context.get("weather"), + "holidays": context.get("holidays"), + "updated_at": context.get("updated_at"), + }, + "meta": { + "coverage": metrics.get("coverage"), + "metrics": metrics.get("metrics"), + "limits": metrics.get("limits", []), + }, + } + @app.post("/predict") def predict_endpoint(body: PredictRequest) -> dict: state = runtime() diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py new file mode 100644 index 0000000..a6879fb --- /dev/null +++ b/tests/test_bootstrap.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from datetime import datetime + +from pulsecast import config + +BOOTSTRAP_KEYS = { + "data_version", + "model_version", + "context_version", + "sensors", + "series", + "context", + "meta", +} + +SENSOR_KEYS = {"id", "name", "lat", "lon", "mean", "p95", "battery"} + + +def _timestamp(value: str) -> datetime: + assert value.endswith("Z") + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def test_bootstrap_shape(client): + response = client.get("/bootstrap") + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("application/json") + payload = response.json() + + assert set(payload) == BOOTSTRAP_KEYS + _timestamp(payload["data_version"]) + _timestamp(payload["context_version"]) + assert payload["model_version"].startswith("histgb-") + + sensors = payload["sensors"] + assert len(sensors) == 8 + assert {sensor["id"] for sensor in sensors} == set(config.USABLE_SENSORS) + assert [sensor["id"] for sensor in sensors] == sorted(sensor["id"] for sensor in sensors) + for sensor in sensors: + assert set(sensor) == SENSOR_KEYS + assert sensor["p95"] >= sensor["mean"] + assert sensor["battery"] is None or 0 <= sensor["battery"] <= 100 + + +def test_bootstrap_series_is_hourly_for_all_eight_sensors(client): + series = client.get("/bootstrap").json()["series"] + assert set(series) == set(config.USABLE_SENSORS) + + total_points = 0 + for sensor, points in series.items(): + assert len(points) > 1000, sensor + total_points += len(points) + previous = None + for point in points: + assert set(point) == {"t", "v"} + timestamp = _timestamp(point["t"]) + assert timestamp.minute == 0 and timestamp.second == 0 + if previous is not None: + assert timestamp > previous + previous = timestamp + assert point["v"] >= 0 + assert total_points > 20_000 + + +def test_bootstrap_context_and_meta(client): + payload = client.get("/bootstrap").json() + + context = payload["context"] + assert set(context) == {"weather", "holidays", "updated_at"} + assert set(context["weather"]) == {"now", "forecast"} + assert len(context["weather"]["forecast"]) == 168 + assert context["holidays"] + assert {"date", "name"} == set(context["holidays"][0]) + _timestamp(context["updated_at"]) + + meta = payload["meta"] + assert set(meta) == {"coverage", "metrics", "limits"} + assert meta["coverage"]["sensors"] == 8 + assert meta["coverage"]["cadence_min"] == 15 + assert meta["metrics"]["mae"] < meta["metrics"]["baseline_mae"] + assert len(meta["limits"]) >= 4 + + +def test_bootstrap_versions_match_meta(client): + payload = client.get("/bootstrap").json() + meta = client.get("/meta").json() + for key in ("data_version", "model_version", "context_version"): + assert payload[key] == meta[key]