diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..230f40f --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,41 @@ +name: ci + +on: + push: + branches: + - main + - "feat/**" + pull_request: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + run: python -m pip install "uv==0.10.4" + + # Same checks as local `make check`, split into steps for readable CI + # failures. No secrets, no image push, no cluster access. + - name: Sync locked environment + run: uv sync --frozen + + - name: Static analysis (ruff) + run: uv run ruff check src tests main.py + + - name: Byte-compile + run: uv run python -m compileall src + + - name: Tests + run: uv run pytest -q diff --git a/Dockerfile b/Dockerfile index 8be7296..113eaea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,17 +24,18 @@ ENV PYTHONUNBUFFERED=1 \ PULSECAST_UPLOADS=/tmp/pulsecast/uploads \ PATH=/app/.venv/bin:$PATH -RUN groupadd --system app \ - && useradd --system --gid app --home-dir /app --shell /usr/sbin/nologin app \ - && mkdir -p /tmp/pulsecast/models /tmp/pulsecast/uploads \ - && chown -R app:app /tmp/pulsecast +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid 10001 --home-dir /app --no-create-home \ + --shell /usr/sbin/nologin app \ + && install -d -o 10001 -g 10001 -m 0755 \ + /tmp/pulsecast /tmp/pulsecast/models /tmp/pulsecast/uploads WORKDIR /app -COPY --chown=app:app --from=builder /app/.venv /app/.venv -COPY --chown=app:app artifacts ./artifacts -COPY --chown=app:app main.py README.md model_card.md ./ +COPY --chown=10001:10001 --from=builder /app/.venv /app/.venv +COPY --chown=10001:10001 artifacts ./artifacts +COPY --chown=10001:10001 main.py README.md model_card.md ./ -USER app +USER 10001:10001 EXPOSE 8000 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ diff --git a/Makefile b/Makefile index e7dfa0d..02d5e44 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,14 @@ UV ?= uv -.PHONY: help data train test serve docker clean +.PHONY: help data train test lint check serve docker clean help: @printf '%s\n' \ 'make data parse/clean the raw CSV, join weather + holidays, export JSON' \ 'make train fit the 28-day holdout model, export metrics/forecasts/model' \ 'make test run pytest' \ + 'make lint run ruff static checks' \ + 'make check CI gate: sync --frozen, ruff, compileall, pytest' \ 'make serve run the FastAPI service on :8000' \ 'make docker build the multi-stage image (pulsecast-ml:local)' \ 'make clean remove caches and generated artifacts (keeps data/raw)' @@ -20,6 +22,15 @@ train: test: $(UV) run pytest +lint: + $(UV) run ruff check src tests main.py + +check: + $(UV) sync --frozen + $(UV) run ruff check src tests main.py + $(UV) run python -m compileall src + $(UV) run pytest -q + serve: $(UV) run uvicorn pulsecast.server:app --host 0.0.0.0 --port 8000 diff --git a/README.md b/README.md index 6c91860..cab3161 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ uv sync make data # needs the raw CSV, first run also fetches Open-Meteo weather make train # writes artifacts/model.joblib, metrics.json, forecasts.json make test +make check # CI gate: sync --frozen, ruff, compileall, pytest make serve # http://localhost:8000 ``` @@ -84,6 +85,7 @@ Shapes follow `CONTRACT.md`. The service is internal; Go exposes it publicly. | `GET` | `/healthz` | `{"ok":true}` | | `GET` | `/readyz` | `{"ok":true,"model":true}`, 503 if no model | | `GET` | `/meta` | Versions, coverage, metrics, features, holiday effect | +| `GET` | `/bootstrap` | Snapshot for Go: sensors, hourly series, context, meta | | `POST` | `/predict` | `{sensor,horizon,scenario,weather?,is_holiday?}` → `points` | | `POST` | `/simulate` | Scenario forecast for all or selected sensors | | `POST` | `/train` | Control token; multipart CSV or JSON path; 202 job | @@ -123,8 +125,8 @@ make docker docker run --rm -p 8000:8000 -e CONTROL_TOKEN=change-me pulsecast-ml:local ``` -Multi-stage `python:3.12-slim`, non-root user, model and artifacts baked in, -writable paths only under `/tmp`. +Multi-stage `python:3.12-slim`, numeric non-root user `10001:10001`, model and +artifacts baked in, writable paths only under `/tmp`. ## Configuration @@ -151,7 +153,7 @@ src/pulsecast/ small.py busy classifier + flexible CSV parser retrain.py runtime forecast retraining from upload server.py FastAPI app and training job manager -tests/ schema, health, prediction, small-training job +tests/ schema, bootstrap, health, prediction, small-training job model_card.md features, split, baseline, ablation, limits, leakage ``` diff --git a/pyproject.toml b/pyproject.toml index e7c3d1c..0e948cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,16 @@ packages = ["src/pulsecast"] testpaths = ["tests"] addopts = "-q" +[tool.ruff] +target-version = "py312" +line-length = 100 + +[tool.ruff.lint] +# Stable, fast static gate: syntax errors, undefined/unused names, imports. +select = ["E4", "E7", "E9", "F"] + [dependency-groups] dev = [ "pytest>=9.2.0", + "ruff>=0.17.5", ] diff --git a/src/pulsecast/server.py b/src/pulsecast/server.py index 4e1e358..e67e4ac 100644 --- a/src/pulsecast/server.py +++ b/src/pulsecast/server.py @@ -1,4 +1,4 @@ -"""FastAPI ML service: /healthz, /readyz, /meta, /predict, /simulate, /train. +"""FastAPI ML service: /healthz, /readyz, /meta, /bootstrap, /predict, /simulate, /train. Shapes follow CONTRACT.md. The service is internal: only Go exposes it publicly, and the training routes require a bearer control token. @@ -36,6 +36,7 @@ log = logging.getLogger(__name__) MAX_UPLOAD_BYTES = 60 * 1024 * 1024 ACTIVE_STATUSES = {"queued", "running"} +SENSOR_FIELDS = ("id", "name", "lat", "lon", "mean", "p95", "battery") def _utc_now() -> str: @@ -48,6 +49,9 @@ class Runtime: metrics: dict | None = None weather: pd.DataFrame | None = None weather_meta: dict | None = None + sensors: list[dict] | None = None + series: dict[str, list[dict]] | None = None + context: dict | None = None small_bundle: dict | None = None small_metrics: dict | None = None model_source: str = "baked" @@ -75,6 +79,38 @@ def _load_runtime(runtime: Runtime) -> None: json.loads(meta_path.read_text()) if meta_path.exists() else None ) + # Baked snapshot artifacts served by GET /bootstrap. Go caches this once at + # startup, so loading them at boot keeps the endpoint a pure in-memory read. + # Project to the contract fields; artifacts carry extra audit fields + # (last_seen) that Go does not need. + sensors_path = artifacts / "sensors.json" + if sensors_path.exists(): + sensors = json.loads(sensors_path.read_text()) + runtime.sensors = [ + {key: sensor.get(key) for key in SENSOR_FIELDS} for sensor in sensors + ] + else: + log.warning("snapshot artifact missing: %s", sensors_path) + + series_path = artifacts / "series.json" + if series_path.exists(): + series = json.loads(series_path.read_text()).get("series") or {} + runtime.series = { + sensor: [ + {"t": point["t"], "v": point["v"]} + for point in series.get(sensor, []) + ] + for sensor in config.USABLE_SENSORS + } + else: + log.warning("snapshot artifact missing: %s", series_path) + + context_path = artifacts / "context.json" + if context_path.exists(): + runtime.context = json.loads(context_path.read_text()) + else: + log.warning("snapshot artifact missing: %s", context_path) + small_path = artifacts / "small_model.joblib" small_metrics_path = artifacts / "small_metrics.json" if small_path.exists(): @@ -468,6 +504,39 @@ def create_app() -> FastAPI: } return payload + @app.get("/bootstrap") + def bootstrap() -> dict: + """Snapshot Go caches at startup: sensors, hourly series, context, meta.""" + state = runtime() + if state.bundle is None or state.metrics is None: + raise HTTPException(503, "model is not loaded") + if state.sensors is None or state.series is None or state.context is None: + raise HTTPException(503, "snapshot artifacts are not loaded") + if len(state.sensors) != len(config.USABLE_SENSORS) or not all( + state.series.get(sensor) for sensor in config.USABLE_SENSORS + ): + raise HTTPException(503, "snapshot is incomplete") + + metrics = state.metrics + context = state.context + return { + "data_version": metrics.get("data_version"), + "model_version": metrics.get("model_version"), + "context_version": metrics.get("context_version"), + "sensors": state.sensors, + "series": state.series, + "context": { + "weather": context.get("weather"), + "holidays": context.get("holidays"), + "updated_at": context.get("updated_at"), + }, + "meta": { + "coverage": metrics.get("coverage"), + "metrics": metrics.get("metrics"), + "limits": metrics.get("limits", []), + }, + } + @app.post("/predict") def predict_endpoint(body: PredictRequest) -> dict: state = runtime() diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py new file mode 100644 index 0000000..97254d9 --- /dev/null +++ b/tests/test_bootstrap.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from datetime import datetime + +from pulsecast import config + +BOOTSTRAP_KEYS = { + "data_version", + "model_version", + "context_version", + "sensors", + "series", + "context", + "meta", +} + +SENSOR_KEYS = {"id", "name", "lat", "lon", "mean", "p95", "battery"} + + +def _timestamp(value: str) -> datetime: + assert value.endswith("Z") + return datetime.fromisoformat(value) + + +def test_bootstrap_shape(client): + response = client.get("/bootstrap") + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("application/json") + payload = response.json() + + assert set(payload) == BOOTSTRAP_KEYS + _timestamp(payload["data_version"]) + _timestamp(payload["context_version"]) + assert payload["model_version"].startswith("histgb-") + + sensors = payload["sensors"] + assert len(sensors) == 8 + assert {sensor["id"] for sensor in sensors} == set(config.USABLE_SENSORS) + assert [sensor["id"] for sensor in sensors] == sorted(sensor["id"] for sensor in sensors) + for sensor in sensors: + assert set(sensor) == SENSOR_KEYS + assert sensor["p95"] >= sensor["mean"] + assert sensor["battery"] is None or 0 <= sensor["battery"] <= 100 + + +def test_bootstrap_series_is_hourly_for_all_eight_sensors(client): + series = client.get("/bootstrap").json()["series"] + assert set(series) == set(config.USABLE_SENSORS) + + total_points = 0 + for sensor, points in series.items(): + assert len(points) > 1000, sensor + total_points += len(points) + previous = None + for point in points: + assert set(point) == {"t", "v"} + timestamp = _timestamp(point["t"]) + assert timestamp.minute == 0 and timestamp.second == 0 + if previous is not None: + assert timestamp > previous + previous = timestamp + assert point["v"] >= 0 + assert total_points > 20_000 + + +def test_bootstrap_context_and_meta(client): + payload = client.get("/bootstrap").json() + + context = payload["context"] + assert set(context) == {"weather", "holidays", "updated_at"} + assert set(context["weather"]) == {"now", "forecast"} + assert len(context["weather"]["forecast"]) == 168 + assert context["holidays"] + assert {"date", "name"} == set(context["holidays"][0]) + _timestamp(context["updated_at"]) + + meta = payload["meta"] + assert set(meta) == {"coverage", "metrics", "limits"} + assert meta["coverage"]["sensors"] == 8 + assert meta["coverage"]["cadence_min"] == 15 + assert meta["metrics"]["mae"] < meta["metrics"]["baseline_mae"] + assert len(meta["limits"]) >= 4 + + +def test_bootstrap_versions_match_meta(client): + payload = client.get("/bootstrap").json() + meta = client.get("/meta").json() + for key in ("data_version", "model_version", "context_version"): + assert payload[key] == meta[key] diff --git a/uv.lock b/uv.lock index 2030220..dc1d1ef 100644 --- a/uv.lock +++ b/uv.lock @@ -441,6 +441,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "pytest" }, + { name = "ruff" }, ] [package.metadata] @@ -457,7 +458,10 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=9.2.0" }] +dev = [ + { name = "pytest", specifier = ">=9.2.0" }, + { name = "ruff", specifier = ">=0.17.5" }, +] [[package]] name = "pyarrow" @@ -734,6 +738,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "ruff" +version = "0.17.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/cc/7b71512eb8b49926efb887cc66c74dd993642943c29eba51a877f6b85692/ruff-0.17.5.tar.gz", hash = "sha256:625e57d80cd43352d12ea81690f3eac46cddc8c0f35026d137fb0f157f94592e", size = 5368382, upload-time = "2026-11-12T17:54:55.04Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/f2/61df414cb4338617290a9023ad09486e207fb9a6d09357a7be34f2e369a4/ruff-0.17.5-py3-none-linux_armv6l.whl", hash = "sha256:d511dd302982e46a285528942fa6557d57730edc4bdb9def95e86a2603d77396", size = 10231309, upload-time = "2026-11-12T17:54:09.314Z" }, + { url = "https://files.pythonhosted.org/packages/d1/72/8ed970f3361f7c8d3f8e1b3b1af27cf086bc4b3a6b62ec20e21a1aaaaff5/ruff-0.17.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b72fbc232e328bdbb9cdddf4f1aedcd61bd1aa64d75fc012114b4add0bd7f26b", size = 10250018, upload-time = "2026-11-12T17:54:11.983Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4b/8b10f6c9d0a18413301ae8dd2af9d180a81bb738b1b731d944a37404c523/ruff-0.17.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6f0db748508f04a04e3fbebcc636a5c8b3bd76274f8ed28f497c44d75c2ea2d7", size = 9504737, upload-time = "2026-11-12T17:54:14.567Z" }, + { url = "https://files.pythonhosted.org/packages/b7/ea/1b3afa3943239dede7a0755ea139bdb51879f0f3f96f92894081d5c7d2e5/ruff-0.17.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f6a7b138edf9cc402ca736a59d9525e6d766e5395760b72e7e146e6251ac963", size = 10168006, upload-time = "2026-11-12T17:54:17.195Z" }, + { url = "https://files.pythonhosted.org/packages/31/08/ee641daa45f83792f91274b65796cb011f1065f588911bfa3909ad9e31e7/ruff-0.17.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e52622ecf3a0df9d03e796fe3eed4aa085adda65520db7e0619cbd2192a97342", size = 10043751, upload-time = "2026-11-12T17:54:19.815Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cf/1c59fdb42d0b5671ad5b160636fab4005ee28db72cb06aef6e3f3c04f377/ruff-0.17.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54c0f67d1be11b55d12c184a819fc229f5eee3ef45c16c4fcad41258e83b4882", size = 10690562, upload-time = "2026-11-12T17:54:22.789Z" }, + { url = "https://files.pythonhosted.org/packages/49/c5/578074716173fb0c1591e9648a9f85c469888d9530cb35b89b757dc1208b/ruff-0.17.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd0dfe7c6fe5a447fce467c58e69893be1d9a7e4378f5b9906432da33d94a97f", size = 11338465, upload-time = "2026-11-12T17:54:25.523Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b9/d6428bcf5ed88f4dc1d6e3417ee350236ed36e7a35f9477b727c00e437bf/ruff-0.17.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ec4ecf40fb4d6e1265b009dbe551727920926fc0f769070c452d6d84b422a74d", size = 10895501, upload-time = "2026-11-12T17:54:28.441Z" }, + { url = "https://files.pythonhosted.org/packages/f2/8a/1e29c0c3785ae7e5cd0d54ee8717a08867ef71545796340be4c653c57e24/ruff-0.17.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40ca8afaac309b41a5b2e09e4e7beb1f824826b93c25697c4d74f573324fba4e", size = 10613231, upload-time = "2026-11-12T17:54:30.862Z" }, + { url = "https://files.pythonhosted.org/packages/32/b6/88944350c14bb90651014d678ad554e233ae2a79c14ad52101845f93d15b/ruff-0.17.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8e85601bd6d9b6c3e3b95c0e0dccdbac1d23c6bcef006d187fbcddaf8225a5df", size = 10562789, upload-time = "2026-11-12T17:54:33.869Z" }, + { url = "https://files.pythonhosted.org/packages/6f/fd/2792e955b0bf54295cdcdddc4bb3f380c0790d7e60e789f528a9e6e48a31/ruff-0.17.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f93d2b7d449ed9545304f6923699ca20d13c675b69f258481709a08ae61076db", size = 10069824, upload-time = "2026-11-12T17:54:36.663Z" }, + { url = "https://files.pythonhosted.org/packages/07/9c/d7cad341b4c19237948a6dd1fcea1c9c39c9d08b70f22f0e157d1236ebd6/ruff-0.17.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0e5c125c5b31dc8efc948d41357496a4c4a312977565ccba740030efdff83580", size = 10038961, upload-time = "2026-11-12T17:54:39.452Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a1/9cd0d4c371ddc70267d3bd557e718e2bdfb94c25e49d25605edfabb764bd/ruff-0.17.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:50dfc3c6b3ebd0a932f11dfd579bd42234b0b7137e68940eb6abdbfdb9d69697", size = 10385012, upload-time = "2026-11-12T17:54:41.808Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a9/35f078ce98d4df318328ea5270382b5b65e680f7953b3db54f8e009c420c/ruff-0.17.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:86bd1f3b83c25f9b8055c6e796f0685b39d3127ce947322098c3108cf82fd586", size = 10793672, upload-time = "2026-11-12T17:54:44.342Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a5/2e136dede2a88aaaf54498bbbdc81bc810f788ef6ef01efb97037d73cd1d/ruff-0.17.5-py3-none-win32.whl", hash = "sha256:e1546609b2ad73f1d6c5c076881b788672b7b3cc3283981182cbdbb6524ed88a", size = 9929908, upload-time = "2026-11-12T17:54:47.067Z" }, + { url = "https://files.pythonhosted.org/packages/ec/8c/e4db9ae1e0ee233c04df007ab6d0d42f799ae9abd95d352b6092ee7bc4bf/ruff-0.17.5-py3-none-win_amd64.whl", hash = "sha256:aa415d57c244299279901d776607fee3df7831c2a7435433fbfacde8632c6550", size = 10275454, upload-time = "2026-11-12T17:54:49.401Z" }, + { url = "https://files.pythonhosted.org/packages/e6/65/a1648bc1550e7683bf7bf1e2ce2485c9f8535cbca6cca9ec7b73cc12fb8e/ruff-0.17.5-py3-none-win_arm64.whl", hash = "sha256:64a6fd668a336df959c522991de33e9e3c8be463afc0c9ac8f029f9d19ef756e", size = 9849020, upload-time = "2026-11-12T17:54:52.055Z" }, +] + [[package]] name = "scikit-learn" version = "1.9.1"