Compare commits
3
Commits
feat/ci-docs
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
603b1c8cee | ||
|
|
ad63d1caf3 | ||
|
|
507f33b7ea |
No files matched your search
@@ -1,12 +1,13 @@
|
|||||||
UV ?= uv
|
UV ?= uv
|
||||||
|
|
||||||
.PHONY: help data train small figures test lint check serve docker clean
|
.PHONY: help data train small prezero figures test lint check serve docker clean
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@printf '%s\n' \
|
@printf '%s\n' \
|
||||||
'make data parse/clean the raw CSV, join weather + holidays, export JSON' \
|
'make data parse/clean the raw CSV, join weather + holidays, export JSON' \
|
||||||
'make train fit the 28-day holdout model, export metrics/forecasts/model' \
|
'make train fit the 28-day holdout model, export metrics/forecasts/model' \
|
||||||
'make small train + print the busy-hour classifier metrics from the raw CSV' \
|
'make small train + print the busy-hour classifier metrics from the raw CSV' \
|
||||||
|
'make prezero build artifacts/prezero_priority.json from the fill-level CSV' \
|
||||||
'make figures regenerate docs/figures/*.svg from artifacts/metrics.json' \
|
'make figures regenerate docs/figures/*.svg from artifacts/metrics.json' \
|
||||||
'make test run pytest' \
|
'make test run pytest' \
|
||||||
'make lint run ruff static checks' \
|
'make lint run ruff static checks' \
|
||||||
@@ -24,6 +25,9 @@ train:
|
|||||||
small:
|
small:
|
||||||
$(UV) run python scripts/train_small.py
|
$(UV) run python scripts/train_small.py
|
||||||
|
|
||||||
|
prezero:
|
||||||
|
$(UV) run python scripts/build_prezero.py
|
||||||
|
|
||||||
figures:
|
figures:
|
||||||
$(UV) run python scripts/plot_metrics.py
|
$(UV) run python scripts/plot_metrics.py
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ To rebuild the artifacts from raw data:
|
|||||||
make data # needs the raw CSV; first run also fetches Open-Meteo weather
|
make data # needs the raw CSV; first run also fetches Open-Meteo weather
|
||||||
make train # writes artifacts/model.joblib, metrics.json, forecasts.json
|
make train # writes artifacts/model.joblib, metrics.json, forecasts.json
|
||||||
make small # prints the busy-hour classifier metrics + confusion matrix
|
make small # prints the busy-hour classifier metrics + confusion matrix
|
||||||
|
make prezero # rebuilds artifacts/prezero_priority.json from the fill-level CSV
|
||||||
make figures # regenerates docs/figures/*.svg from metrics.json
|
make figures # regenerates docs/figures/*.svg from metrics.json
|
||||||
make test
|
make test
|
||||||
```
|
```
|
||||||
@@ -67,6 +68,11 @@ The raw challenge CSV is **never committed**. The default path is
|
|||||||
git-ignored. The committed `artifacts/` (JSON + joblib + compact weather frames)
|
git-ignored. The committed `artifacts/` (JSON + joblib + compact weather frames)
|
||||||
let the API and Docker image run without the raw file or network.
|
let the API and Docker image run without the raw file or network.
|
||||||
|
|
||||||
|
The PreZero fill-level export is also git-ignored. For `make prezero`, put
|
||||||
|
`glass_fill_levels_anonymized_f.csv` at `data/raw/` or point
|
||||||
|
`PULSECAST_PREZERO_CSV` at it. Only the derived
|
||||||
|
`artifacts/prezero_priority.json` is committed.
|
||||||
|
|
||||||
## Artifacts
|
## Artifacts
|
||||||
|
|
||||||
| File | Purpose |
|
| File | Purpose |
|
||||||
@@ -78,6 +84,7 @@ let the API and Docker image run without the raw file or network.
|
|||||||
| `artifacts/small_metrics.json` | Busy classifier: metrics, confusion matrix, per-sensor, thresholds |
|
| `artifacts/small_metrics.json` | Busy classifier: metrics, confusion matrix, per-sensor, thresholds |
|
||||||
| `artifacts/context.json` | Current + 7-day weather, BW holidays |
|
| `artifacts/context.json` | Current + 7-day weather, BW holidays |
|
||||||
| `artifacts/data_audit.json` | Row/device checks for the raw CSV |
|
| `artifacts/data_audit.json` | Row/device checks for the raw CSV |
|
||||||
|
| `artifacts/prezero_priority.json` | 449-container priority snapshot + last 10 readings each |
|
||||||
| `artifacts/model.joblib` | Baked HistGB forecast bundle |
|
| `artifacts/model.joblib` | Baked HistGB forecast bundle |
|
||||||
| `artifacts/small_model.joblib` | Baked busy classifier |
|
| `artifacts/small_model.joblib` | Baked busy classifier |
|
||||||
| `artifacts/weather_*.parquet` | Baked weather frames for retraining/inference |
|
| `artifacts/weather_*.parquet` | Baked weather frames for retraining/inference |
|
||||||
@@ -93,6 +100,9 @@ exposes it publicly.
|
|||||||
| `GET` | `/readyz` | `{"ok":true,"model":true}`, 503 if no model |
|
| `GET` | `/readyz` | `{"ok":true,"model":true}`, 503 if no model |
|
||||||
| `GET` | `/meta` | Versions, coverage, metrics, features, holiday effect |
|
| `GET` | `/meta` | Versions, coverage, metrics, features, holiday effect |
|
||||||
| `GET` | `/bootstrap` | Snapshot for Go: sensors, hourly series, context, meta |
|
| `GET` | `/bootstrap` | Snapshot for Go: sensors, hourly series, context, meta |
|
||||||
|
| `GET` | `/prezero/summary` | Priority counts and material totals |
|
||||||
|
| `GET` | `/prezero/priority` | Ranked containers; `priority`, `limit`, `offset` |
|
||||||
|
| `GET` | `/prezero/containers/{id}` | One container + its 10 newest readings |
|
||||||
| `POST` | `/predict` | `{sensor,horizon,scenario,weather?,is_holiday?}` → `points` |
|
| `POST` | `/predict` | `{sensor,horizon,scenario,weather?,is_holiday?}` → `points` |
|
||||||
| `POST` | `/simulate` | Scenario forecast for all or selected sensors |
|
| `POST` | `/simulate` | Scenario forecast for all or selected sensors |
|
||||||
| `POST` | `/train` | Control token; multipart CSV or JSON path; 202 job |
|
| `POST` | `/train` | Control token; multipart CSV or JSON path; 202 job |
|
||||||
@@ -125,6 +135,47 @@ and persists to `PULSECAST_MODELS` (default `/tmp/pulsecast/models`). A restart
|
|||||||
falls back to the baked model. `model=small` is the fast demo; `model=forecast`
|
falls back to the baked model. `model=small` is the fast demo; `model=forecast`
|
||||||
retrains the global HistGB (slower, still background).
|
retrains the global HistGB (slower, still background).
|
||||||
|
|
||||||
|
## PreZero container priority
|
||||||
|
|
||||||
|
`artifacts/prezero_priority.json` ranks the 449 anonymous container locations in
|
||||||
|
the 11,701-row glass fill-level export (2025-10-17 → 2026-09-30, 1,338
|
||||||
|
location/material series). The rule uses only reported states:
|
||||||
|
|
||||||
|
- **high:** `overfull_count >= 5` or (`overfull_count >= 2` and
|
||||||
|
`overfull_rate >= 0.25`)
|
||||||
|
- **medium:** `overfull_count >= 1`, not high
|
||||||
|
- **low:** `overfull_count == 0`
|
||||||
|
|
||||||
|
The generated snapshot has **high 88, medium 248, low 113**, with 1,192 overfull
|
||||||
|
readings. Sorted by tier, overfull count/rate, staleness (`reference_date` is
|
||||||
|
the dataset max date, 2026-09-30, not today), then location id. The artifact
|
||||||
|
also holds each container's 10 newest readings for the history route.
|
||||||
|
|
||||||
|
**The dataset has no coordinates.** This is a service-priority proof, not a
|
||||||
|
route; there is no capacity or depot data, and `overfull` is a reported state,
|
||||||
|
not a measured volume. The committed artifact is the API's source of truth. If
|
||||||
|
it is missing, only `/prezero/*` returns 503; the forecast/training routes keep
|
||||||
|
working.
|
||||||
|
|
||||||
|
Rebuild from the raw, git-ignored export:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp glass_fill_levels_anonymized_f.csv data/raw/ # or set PULSECAST_PREZERO_CSV
|
||||||
|
make prezero
|
||||||
|
# equivalently:
|
||||||
|
uv run python scripts/build_prezero.py \
|
||||||
|
--csv data/raw/glass_fill_levels_anonymized_f.csv \
|
||||||
|
--out artifacts/prezero_priority.json
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s localhost:8000/prezero/summary
|
||||||
|
curl -s 'localhost:8000/prezero/priority?priority=high&limit=3'
|
||||||
|
curl -s localhost:8000/prezero/containers/<location_id>
|
||||||
|
```
|
||||||
|
|
||||||
|
Detail, counts, and limitations: [`docs/prezero.md`](docs/prezero.md).
|
||||||
|
|
||||||
## End-to-end check
|
## End-to-end check
|
||||||
|
|
||||||
`scripts/e2e_train.py` starts a real uvicorn process on port 18000 with a
|
`scripts/e2e_train.py` starts a real uvicorn process on port 18000 with a
|
||||||
@@ -145,6 +196,7 @@ The unit and contract tests are `make test`; `make check` is the CI gate.
|
|||||||
| `docs/metrics.md` | Paper tables/figures: protocol, headline, ablation, per sensor, uncertainty, small model, claim limits |
|
| `docs/metrics.md` | Paper tables/figures: protocol, headline, ablation, per sensor, uncertainty, small model, claim limits |
|
||||||
| `docs/small-model.md` | Busy-hour target, thresholds, split, metrics, confusion matrix, per sensor, limits |
|
| `docs/small-model.md` | Busy-hour target, thresholds, split, metrics, confusion matrix, per sensor, limits |
|
||||||
| `docs/train-e2e.md` | Exact live training-job evidence (small + forecast, hot-swap, errors) |
|
| `docs/train-e2e.md` | Exact live training-job evidence (small + forecast, hot-swap, errors) |
|
||||||
|
| `docs/prezero.md` | Container-priority rule, counts, artifact/API shape, no-coordinates limits |
|
||||||
| `docs/figures/*.svg` | Ablation MAE, per-sensor MAE, small-model confusion matrix (generated) |
|
| `docs/figures/*.svg` | Ablation MAE, per-sensor MAE, small-model confusion matrix (generated) |
|
||||||
| `model_card.md` | Model card: features, split, baseline, ablation, leakage, limitations |
|
| `model_card.md` | Model card: features, split, baseline, ablation, leakage, limitations |
|
||||||
|
|
||||||
@@ -167,6 +219,8 @@ artifacts baked in, writable paths only under `/tmp`.
|
|||||||
| `PULSECAST_ARTIFACTS` | `./artifacts` | Baked JSON/model output |
|
| `PULSECAST_ARTIFACTS` | `./artifacts` | Baked JSON/model output |
|
||||||
| `PULSECAST_CACHE` | `./data/cache` | Parquet/weather cache |
|
| `PULSECAST_CACHE` | `./data/cache` | Parquet/weather cache |
|
||||||
| `PULSECAST_WEATHER` | `PULSECAST_ARTIFACTS` | Baked weather frame location |
|
| `PULSECAST_WEATHER` | `PULSECAST_ARTIFACTS` | Baked weather frame location |
|
||||||
|
| `PULSECAST_PREZERO_ARTIFACT` | `./artifacts/prezero_priority.json` | PreZero priority snapshot |
|
||||||
|
| `PULSECAST_PREZERO_CSV` | `./data/raw/glass_fill_levels_anonymized_f.csv` | Raw fill-level export for `make prezero` |
|
||||||
| `PULSECAST_MODELS` | `/tmp/pulsecast/models` | Retrained model persistence |
|
| `PULSECAST_MODELS` | `/tmp/pulsecast/models` | Retrained model persistence |
|
||||||
| `PULSECAST_UPLOADS` | `/tmp/pulsecast/uploads` | Multipart upload landing |
|
| `PULSECAST_UPLOADS` | `/tmp/pulsecast/uploads` | Multipart upload landing |
|
||||||
| `CONTROL_TOKEN` | unset | Required by `/train`; service returns 503 if unset |
|
| `CONTROL_TOKEN` | unset | Required by `/train`; service returns 503 if unset |
|
||||||
@@ -184,10 +238,11 @@ src/pulsecast/
|
|||||||
train.py `make train`, ablation, artifacts
|
train.py `make train`, ablation, artifacts
|
||||||
small.py busy classifier + flexible CSV parser
|
small.py busy classifier + flexible CSV parser
|
||||||
retrain.py runtime forecast retraining from upload
|
retrain.py runtime forecast retraining from upload
|
||||||
|
prezero.py load/validate/filter/page/lookup the priority snapshot
|
||||||
server.py FastAPI app and training job manager
|
server.py FastAPI app and training job manager
|
||||||
scripts/ train_small.py, plot_metrics.py, e2e_train.py
|
scripts/ train_small.py, plot_metrics.py, build_prezero.py, e2e_train.py
|
||||||
docs/ metrics.md, small-model.md, train-e2e.md, generated figures
|
docs/ metrics.md, small-model.md, train-e2e.md, prezero.md, figures
|
||||||
tests/ schema, bootstrap, health, prediction, small-training job
|
tests/ schema, bootstrap, health, prediction, prezero, small-training job
|
||||||
model_card.md features, split, baseline, ablation, limits, leakage
|
model_card.md features, split, baseline, ablation, limits, leakage
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -200,3 +255,5 @@ model_card.md features, split, baseline, ablation, limits, leakage
|
|||||||
- Holidays are context and a small feature, not a large accuracy claim.
|
- Holidays are context and a small feature, not a large accuracy claim.
|
||||||
- No POS, sales, queue, or labor-cost data; staffing is decision support with
|
- No POS, sales, queue, or labor-cost data; staffing is decision support with
|
||||||
explicit assumptions.
|
explicit assumptions.
|
||||||
|
- PreZero has no coordinates, capacity, or depot data: it ranks collection
|
||||||
|
priority, it does not produce or optimize routes.
|
||||||
File diff suppressed because it is too large.
Load diff
+179
@@ -0,0 +1,179 @@
|
|||||||
|
# PreZero container priority
|
||||||
|
|
||||||
|
A transparent collection-priority snapshot built from the anonymized glass
|
||||||
|
fill-level export. The committed artifact is
|
||||||
|
`artifacts/prezero_priority.json`; the raw CSV is **not** committed.
|
||||||
|
|
||||||
|
**This is a priority proof, not a route.** The dataset has no coordinates, no
|
||||||
|
container capacity, and no depot data. `overfull` is a reported state, not a
|
||||||
|
measured volume.
|
||||||
|
|
||||||
|
## Provenance
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|---|---:|
|
||||||
|
| Input | `glass_fill_levels_anonymized_f.csv` (UTF-8 with BOM) |
|
||||||
|
| Rows | 11,701 |
|
||||||
|
| Distinct `Location_id` | 449 |
|
||||||
|
| `Location_id` + `material_type` series | 1,338 |
|
||||||
|
| Date range | 2025-10-17 → 2026-09-30 |
|
||||||
|
| `overfull` readings | 1,192 |
|
||||||
|
| Material readings | white 4,432 · green 4,169 · brown 3,100 |
|
||||||
|
| Material overfull | white 731 · green 357 · brown 104 |
|
||||||
|
|
||||||
|
The export is read with `utf-8-sig`; header names and cells are stripped. Every
|
||||||
|
row must have `Location_id`, `material_type`, a `YYYY-MM-DD` date, and a
|
||||||
|
`fill_level_perc` of either `overfull` or an integer 0–100.
|
||||||
|
|
||||||
|
The raw file stays git-ignored under `data/raw/`. Put it there or set
|
||||||
|
`PULSECAST_PREZERO_CSV`; the generated artifact is committed so the API and
|
||||||
|
Docker image do not need the CSV.
|
||||||
|
|
||||||
|
## Rule
|
||||||
|
|
||||||
|
Each `Location_id` is one container location. For all rows at that location:
|
||||||
|
|
||||||
|
- `readings`, `overfull_count`, `overfull_rate = overfull_count / readings`
|
||||||
|
- `numeric_count` = rows with a numeric 0–100 fill (excludes `overfull`)
|
||||||
|
- `p95_fill` = 95th percentile of numeric readings, `null` if none
|
||||||
|
- `max_fill`, `mean_fill` = numeric readings only
|
||||||
|
- `last_fill`, `last_is_overfull`, `last_date` = most recent row
|
||||||
|
- `days_since_last` = `reference_date - last_date`; reference is the dataset
|
||||||
|
max date (2026-09-30), not today
|
||||||
|
- `trend` = least-squares slope over the last up to 3 numeric readings
|
||||||
|
(percentage points per reading); `null` with fewer than 2
|
||||||
|
- `priority_score = round(overfull_count + 2 * overfull_rate, 4)` — a display
|
||||||
|
and sorting convenience, not the tier
|
||||||
|
- `materials` = per material: readings, overfull count, last fill/state/date
|
||||||
|
|
||||||
|
Tier:
|
||||||
|
|
||||||
|
- **high:** `overfull_count >= 5` OR (`overfull_count >= 2` AND
|
||||||
|
`overfull_rate >= 0.25`)
|
||||||
|
- **medium:** `overfull_count >= 1` and not high
|
||||||
|
- **low:** `overfull_count == 0`
|
||||||
|
|
||||||
|
Result on the attached CSV: **high 88, medium 248, low 113**.
|
||||||
|
|
||||||
|
Order: tier (`high` first), then overfull count desc, overfull rate desc,
|
||||||
|
`days_since_last` asc, `location_id` asc.
|
||||||
|
|
||||||
|
## Artifact shape
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"meta": {
|
||||||
|
"source": "glass_fill_levels_anonymized_f.csv",
|
||||||
|
"generated_at": "2026-11-14T17:00:00Z",
|
||||||
|
"rows": 11701,
|
||||||
|
"containers": 449,
|
||||||
|
"material_series": 1338,
|
||||||
|
"date_from": "2025-10-17",
|
||||||
|
"date_to": "2026-09-30",
|
||||||
|
"reference_date": "2026-09-30",
|
||||||
|
"overfull_readings": 1192,
|
||||||
|
"priority_rule": "high: ...; medium: ...; low: 0",
|
||||||
|
"limitations": ["No coordinates ...", "..."]
|
||||||
|
},
|
||||||
|
"summary": {
|
||||||
|
"priority_counts": {"high": 88, "medium": 248, "low": 113},
|
||||||
|
"by_material": {
|
||||||
|
"white": {"readings": 4432, "overfull": 731},
|
||||||
|
"green": {"readings": 4169, "overfull": 357},
|
||||||
|
"brown": {"readings": 3100, "overfull": 104}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"containers": [
|
||||||
|
{
|
||||||
|
"location_id": "...",
|
||||||
|
"priority": "high",
|
||||||
|
"priority_score": 21.525,
|
||||||
|
"readings": 80,
|
||||||
|
"overfull_count": 21,
|
||||||
|
"overfull_rate": 0.2625,
|
||||||
|
"numeric_count": 59,
|
||||||
|
"p95_fill": 100.0,
|
||||||
|
"max_fill": 100,
|
||||||
|
"mean_fill": 64.24,
|
||||||
|
"last_fill": null,
|
||||||
|
"last_is_overfull": true,
|
||||||
|
"last_date": "2026-09-29",
|
||||||
|
"days_since_last": 1,
|
||||||
|
"trend": 30.0,
|
||||||
|
"materials": {"white": {"readings": 29, "...": "..."}},
|
||||||
|
"recent_readings": [
|
||||||
|
{"date": "2026-09-29", "material_type": "white",
|
||||||
|
"fill_level_perc": "overfull", "is_overfull": true}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The artifact holds at most 10 recent readings per container, not the full
|
||||||
|
11,701-row table. `last_fill` is `null` when the newest reading is `overfull`.
|
||||||
|
`p95_fill`, `max_fill`, `mean_fill`, and `trend` are `null` when the location
|
||||||
|
has no numeric readings or too few for a slope.
|
||||||
|
|
||||||
|
## Generation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Raw CSV is git-ignored; copy it or set PULSECAST_PREZERO_CSV.
|
||||||
|
cp glass_fill_levels_anonymized_f.csv data/raw/
|
||||||
|
make prezero
|
||||||
|
|
||||||
|
# Equivalent explicit command:
|
||||||
|
uv run python scripts/build_prezero.py \
|
||||||
|
--csv data/raw/glass_fill_levels_anonymized_f.csv \
|
||||||
|
--out artifacts/prezero_priority.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The script prints rows, containers, material series, overfull readings, date
|
||||||
|
range, and the three tier counts. It fails loudly on missing columns, blank
|
||||||
|
cells, bad dates, or invalid fill values instead of guessing.
|
||||||
|
|
||||||
|
## Internal API
|
||||||
|
|
||||||
|
The ML service loads and validates the artifact at startup. If it is missing or
|
||||||
|
malformed, only these routes return 503; `/healthz`, `/readyz`, `/bootstrap`,
|
||||||
|
`/predict`, `/simulate`, and `/meta` keep working.
|
||||||
|
|
||||||
|
| Method | Path | Behavior |
|
||||||
|
|---|---|---|
|
||||||
|
| GET | `/prezero/summary` | `meta` + `summary` |
|
||||||
|
| GET | `/prezero/priority` | Ranked containers; `priority` (`high\|medium\|low`), `limit` 1–500 (default 50), `offset` >= 0 |
|
||||||
|
| GET | `/prezero/containers/{location_id}` | One container + 10 newest readings; unknown → 404 |
|
||||||
|
|
||||||
|
Invalid `priority`, `limit`, or `offset` → 400 in the existing error envelope.
|
||||||
|
The priority list omits `recent_readings`; the container route returns it as
|
||||||
|
`readings`. `/prezero/priority` reports `total` before pagination and `count`
|
||||||
|
after it.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s localhost:8000/prezero/summary
|
||||||
|
curl -s 'localhost:8000/prezero/priority?priority=high&limit=3'
|
||||||
|
curl -s localhost:8000/prezero/containers/<known-id>
|
||||||
|
```
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run pytest -q tests/test_prezero.py
|
||||||
|
make check
|
||||||
|
```
|
||||||
|
|
||||||
|
The tests assert the dataset counts above (11,701 rows, 449 containers, 1,338
|
||||||
|
series, 1,192 overfull, 88/248/113 tiers), filtering/pagination, known and
|
||||||
|
unknown containers, invalid queries, artifact schema and tier consistency, and
|
||||||
|
the missing-artifact 503 path.
|
||||||
|
|
||||||
|
## Limits
|
||||||
|
|
||||||
|
- No coordinates: cannot compute distance, travel time, or a route.
|
||||||
|
- No capacity, depot, vehicle, or shift data.
|
||||||
|
- `overfull` is a reported state; fill percentages are coarse, user-reported
|
||||||
|
values (0/10/.../100), not continuous sensor volumes.
|
||||||
|
- The reference date is the dataset max date, so staleness is relative to the
|
||||||
|
snapshot, not to when the API is called.
|
||||||
|
- Counts and rates describe this export only; they are not a forecast of future
|
||||||
|
fill levels.
|
||||||
@@ -0,0 +1,329 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Build ``artifacts/prezero_priority.json`` from the raw fill-level export.
|
||||||
|
|
||||||
|
The raw CSV is challenge data and stays git-ignored. The derived artifact is
|
||||||
|
committed and is the source of truth for the internal ``/prezero`` routes.
|
||||||
|
|
||||||
|
Priority rule (contract, transparent, no invented labels):
|
||||||
|
|
||||||
|
high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25)
|
||||||
|
medium: overfull_count >= 1 and not high
|
||||||
|
low: overfull_count == 0
|
||||||
|
|
||||||
|
There are no coordinates in the dataset: this is a service-priority proof, not
|
||||||
|
a route. ``priority_score`` is a sorting/display convenience; the tier is the
|
||||||
|
contract.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
uv run python scripts/build_prezero.py
|
||||||
|
uv run python scripts/build_prezero.py --csv data/raw/glass_fill_levels_anonymized_f.csv
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from datetime import UTC, date, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from pulsecast import config
|
||||||
|
from pulsecast.prezero import container_sort_key, priority_tier
|
||||||
|
|
||||||
|
REQUIRED_COLUMNS = ("Location_id", "material_type", "date", "fill_level_perc")
|
||||||
|
MATERIAL_ORDER = ("white", "green", "brown")
|
||||||
|
PRIORITY_RULE = (
|
||||||
|
"high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25); "
|
||||||
|
"medium: overfull_count >= 1 and not high; low: 0"
|
||||||
|
)
|
||||||
|
LIMITATIONS = [
|
||||||
|
"No coordinates in the dataset; this is a priority proof, not a route.",
|
||||||
|
"No container capacity or depot data.",
|
||||||
|
"Overfull is a reported state, not a measured volume.",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class BuildError(ValueError):
|
||||||
|
"""The input CSV does not satisfy the PreZero contract."""
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_columns(fieldnames: list[str] | None, path: Path) -> dict[str, str]:
|
||||||
|
"""Map BOM/whitespace-normalized names back to the CSV's raw names."""
|
||||||
|
if not fieldnames:
|
||||||
|
raise BuildError(f"{path}: missing header row")
|
||||||
|
mapping: dict[str, str] = {}
|
||||||
|
for raw in fieldnames:
|
||||||
|
name = str(raw).strip().lstrip("\ufeff")
|
||||||
|
if not name:
|
||||||
|
raise BuildError(f"{path}: blank column name")
|
||||||
|
if name in mapping:
|
||||||
|
raise BuildError(f"{path}: duplicate column after trimming: {name}")
|
||||||
|
mapping[name] = raw
|
||||||
|
missing = [name for name in REQUIRED_COLUMNS if name not in mapping]
|
||||||
|
if missing:
|
||||||
|
raise BuildError(
|
||||||
|
f"{path}: missing required columns: {', '.join(missing)}; "
|
||||||
|
f"found: {', '.join(mapping)}"
|
||||||
|
)
|
||||||
|
return mapping
|
||||||
|
|
||||||
|
|
||||||
|
def _read_rows(path: Path) -> list[dict[str, Any]]:
|
||||||
|
"""Read the export as strings, normalize headers/cells, validate every row."""
|
||||||
|
try:
|
||||||
|
handle = path.open("r", encoding="utf-8-sig", newline="")
|
||||||
|
except FileNotFoundError as exc:
|
||||||
|
raise BuildError(
|
||||||
|
f"CSV not found: {path}. Copy the raw export there or pass --csv."
|
||||||
|
) from exc
|
||||||
|
except OSError as exc:
|
||||||
|
raise BuildError(f"cannot read {path}: {exc}") from exc
|
||||||
|
|
||||||
|
rows: list[dict[str, Any]] = []
|
||||||
|
with handle:
|
||||||
|
reader = csv.DictReader(handle)
|
||||||
|
columns = _normalized_columns(reader.fieldnames, path)
|
||||||
|
for line_number, raw_row in enumerate(reader, start=2):
|
||||||
|
row = {
|
||||||
|
name: str(raw_row.get(source) or "").strip()
|
||||||
|
for name, source in columns.items()
|
||||||
|
}
|
||||||
|
blank = [name for name in REQUIRED_COLUMNS if not row[name]]
|
||||||
|
if blank:
|
||||||
|
raise BuildError(f"{path}:{line_number}: blank values in {', '.join(blank)}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
reading_date = date.fromisoformat(row["date"])
|
||||||
|
except ValueError as exc:
|
||||||
|
raise BuildError(
|
||||||
|
f"{path}:{line_number}: invalid date {row['date']!r}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
fill_text = row["fill_level_perc"]
|
||||||
|
if fill_text == "overfull":
|
||||||
|
fill: int | None = None
|
||||||
|
is_overfull = True
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
fill = int(fill_text)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise BuildError(
|
||||||
|
f"{path}:{line_number}: fill_level_perc must be an integer "
|
||||||
|
f"0-100 or 'overfull', got {fill_text!r}"
|
||||||
|
) from exc
|
||||||
|
if not 0 <= fill <= 100:
|
||||||
|
raise BuildError(
|
||||||
|
f"{path}:{line_number}: fill_level_perc out of range: {fill}"
|
||||||
|
)
|
||||||
|
is_overfull = False
|
||||||
|
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"index": len(rows),
|
||||||
|
"location_id": row["Location_id"],
|
||||||
|
"material_type": row["material_type"],
|
||||||
|
"date": row["date"],
|
||||||
|
"date_value": reading_date,
|
||||||
|
"fill_level_perc": fill_text,
|
||||||
|
"fill": fill,
|
||||||
|
"is_overfull": is_overfull,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if not rows:
|
||||||
|
raise BuildError(f"{path}: no data rows")
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _ordered(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
"""Chronological; input order breaks ties on the same date."""
|
||||||
|
return sorted(rows, key=lambda row: (row["date_value"], row["index"]))
|
||||||
|
|
||||||
|
|
||||||
|
def _trend(numeric: list[int]) -> float | None:
|
||||||
|
"""Least-squares slope over the last up to 3 numeric readings (points/read)."""
|
||||||
|
if len(numeric) < 2:
|
||||||
|
return None
|
||||||
|
tail = numeric[-3:]
|
||||||
|
x_mean = (len(tail) - 1) / 2
|
||||||
|
y_mean = sum(tail) / len(tail)
|
||||||
|
numerator = sum((x - x_mean) * (y - y_mean) for x, y in enumerate(tail))
|
||||||
|
denominator = sum((x - x_mean) ** 2 for x in range(len(tail)))
|
||||||
|
slope = numerator / denominator
|
||||||
|
return round(slope, 2) if slope else 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _material_names(materials: set[str]) -> list[str]:
|
||||||
|
known = [name for name in MATERIAL_ORDER if name in materials]
|
||||||
|
return known + sorted(materials - set(known))
|
||||||
|
|
||||||
|
|
||||||
|
def _material_summary(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||||
|
ordered = _ordered(rows)
|
||||||
|
last = ordered[-1]
|
||||||
|
return {
|
||||||
|
"readings": len(ordered),
|
||||||
|
"overfull_count": sum(row["is_overfull"] for row in ordered),
|
||||||
|
"last_fill": last["fill"],
|
||||||
|
"last_is_overfull": last["is_overfull"],
|
||||||
|
"last_date": last["date"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _container(rows: list[dict[str, Any]], reference: date) -> dict[str, Any]:
|
||||||
|
ordered = _ordered(rows)
|
||||||
|
readings = len(ordered)
|
||||||
|
overfull_count = sum(row["is_overfull"] for row in ordered)
|
||||||
|
overfull_rate = overfull_count / readings
|
||||||
|
numeric = [row["fill"] for row in ordered if row["fill"] is not None]
|
||||||
|
last = ordered[-1]
|
||||||
|
|
||||||
|
by_material: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||||
|
for row in ordered:
|
||||||
|
by_material[row["material_type"]].append(row)
|
||||||
|
materials = {
|
||||||
|
name: _material_summary(by_material[name])
|
||||||
|
for name in _material_names(set(by_material))
|
||||||
|
}
|
||||||
|
|
||||||
|
p95_fill = round(float(np.percentile(numeric, 95)), 2) if numeric else None
|
||||||
|
mean_fill = round(sum(numeric) / len(numeric), 2) if numeric else None
|
||||||
|
recent = [
|
||||||
|
{
|
||||||
|
"date": row["date"],
|
||||||
|
"material_type": row["material_type"],
|
||||||
|
"fill_level_perc": row["fill_level_perc"],
|
||||||
|
"is_overfull": row["is_overfull"],
|
||||||
|
}
|
||||||
|
for row in reversed(ordered[-10:])
|
||||||
|
]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"location_id": ordered[0]["location_id"],
|
||||||
|
"priority": priority_tier(overfull_count, overfull_rate),
|
||||||
|
"priority_score": round(overfull_count + 2 * overfull_rate, 4),
|
||||||
|
"readings": readings,
|
||||||
|
"overfull_count": overfull_count,
|
||||||
|
"overfull_rate": overfull_rate,
|
||||||
|
"numeric_count": len(numeric),
|
||||||
|
"p95_fill": p95_fill,
|
||||||
|
"max_fill": max(numeric) if numeric else None,
|
||||||
|
"mean_fill": mean_fill,
|
||||||
|
"last_fill": last["fill"],
|
||||||
|
"last_is_overfull": last["is_overfull"],
|
||||||
|
"last_date": last["date"],
|
||||||
|
"days_since_last": (reference - last["date_value"]).days,
|
||||||
|
"trend": _trend(numeric),
|
||||||
|
"materials": materials,
|
||||||
|
"recent_readings": recent,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build(csv_path: Path) -> dict[str, Any]:
|
||||||
|
rows = _read_rows(csv_path)
|
||||||
|
reference = max(row["date_value"] for row in rows)
|
||||||
|
date_from = min(row["date_value"] for row in rows)
|
||||||
|
|
||||||
|
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||||
|
for row in rows:
|
||||||
|
grouped[row["location_id"]].append(row)
|
||||||
|
containers = [
|
||||||
|
_container(grouped[location_id], reference)
|
||||||
|
for location_id in sorted(grouped)
|
||||||
|
]
|
||||||
|
containers.sort(key=container_sort_key)
|
||||||
|
|
||||||
|
material_counts = Counter(row["material_type"] for row in rows)
|
||||||
|
material_overfull = Counter(
|
||||||
|
row["material_type"] for row in rows if row["is_overfull"]
|
||||||
|
)
|
||||||
|
priority_counts = Counter(container["priority"] for container in containers)
|
||||||
|
by_material = {
|
||||||
|
name: {
|
||||||
|
"readings": material_counts[name],
|
||||||
|
"overfull": material_overfull[name],
|
||||||
|
}
|
||||||
|
for name in _material_names(set(material_counts))
|
||||||
|
}
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
"source": csv_path.name,
|
||||||
|
"generated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
"rows": len(rows),
|
||||||
|
"containers": len(containers),
|
||||||
|
"material_series": len(
|
||||||
|
{(row["location_id"], row["material_type"]) for row in rows}
|
||||||
|
),
|
||||||
|
"date_from": date_from.isoformat(),
|
||||||
|
"date_to": reference.isoformat(),
|
||||||
|
"reference_date": reference.isoformat(),
|
||||||
|
"overfull_readings": sum(row["is_overfull"] for row in rows),
|
||||||
|
"priority_rule": PRIORITY_RULE,
|
||||||
|
"limitations": list(LIMITATIONS),
|
||||||
|
}
|
||||||
|
summary = {
|
||||||
|
"priority_counts": {
|
||||||
|
"high": priority_counts["high"],
|
||||||
|
"medium": priority_counts["medium"],
|
||||||
|
"low": priority_counts["low"],
|
||||||
|
},
|
||||||
|
"by_material": by_material,
|
||||||
|
}
|
||||||
|
return {"meta": meta, "summary": summary, "containers": containers}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description=__doc__,
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--csv",
|
||||||
|
type=Path,
|
||||||
|
default=config.prezero_csv(),
|
||||||
|
help="raw fill-level CSV (git-ignored; default: %(default)s)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--out",
|
||||||
|
type=Path,
|
||||||
|
default=config.prezero_artifact(),
|
||||||
|
help="artifact destination (default: %(default)s)",
|
||||||
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
args = _parse_args(argv)
|
||||||
|
try:
|
||||||
|
payload = build(args.csv)
|
||||||
|
except BuildError as exc:
|
||||||
|
print(f"build_prezero: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.out.write_text(
|
||||||
|
json.dumps(payload, indent=2, ensure_ascii=False) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
meta = payload["meta"]
|
||||||
|
counts = payload["summary"]["priority_counts"]
|
||||||
|
print(f"wrote {args.out} ({args.out.stat().st_size:,} bytes)")
|
||||||
|
print(
|
||||||
|
f"rows={meta['rows']} containers={meta['containers']} "
|
||||||
|
f"material_series={meta['material_series']} "
|
||||||
|
f"overfull={meta['overfull_readings']}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"date_range={meta['date_from']}..{meta['date_to']} "
|
||||||
|
f"high={counts['high']} medium={counts['medium']} low={counts['low']}"
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -45,6 +45,26 @@ def raw_csv_path() -> Path:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def prezero_artifact() -> Path:
|
||||||
|
"""Committed PreZero priority snapshot served by the /prezero routes."""
|
||||||
|
return Path(
|
||||||
|
os.environ.get(
|
||||||
|
"PULSECAST_PREZERO_ARTIFACT",
|
||||||
|
artifacts_dir() / "prezero_priority.json",
|
||||||
|
)
|
||||||
|
).resolve()
|
||||||
|
|
||||||
|
|
||||||
|
def prezero_csv() -> Path:
|
||||||
|
"""Raw fill-level export used only to rebuild the PreZero artifact."""
|
||||||
|
return Path(
|
||||||
|
os.environ.get(
|
||||||
|
"PULSECAST_PREZERO_CSV",
|
||||||
|
_root() / "data" / "raw" / "glass_fill_levels_anonymized_f.csv",
|
||||||
|
)
|
||||||
|
).resolve()
|
||||||
|
|
||||||
|
|
||||||
def weather_dir() -> Path:
|
def weather_dir() -> Path:
|
||||||
"""Baked weather frames live beside the other exported artifacts."""
|
"""Baked weather frames live beside the other exported artifacts."""
|
||||||
return Path(os.environ.get("PULSECAST_WEATHER", artifacts_dir())).resolve()
|
return Path(os.environ.get("PULSECAST_WEATHER", artifacts_dir())).resolve()
|
||||||
|
|||||||
@@ -0,0 +1,384 @@
|
|||||||
|
"""PreZero container-priority snapshot: load, validate, filter, page, lookup.
|
||||||
|
|
||||||
|
The committed ``artifacts/prezero_priority.json`` is the API's source of truth.
|
||||||
|
The dataset has no coordinates, so this ranks collection priority; it does not
|
||||||
|
produce routes. Tiers follow the contract exactly:
|
||||||
|
|
||||||
|
- high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25)
|
||||||
|
- medium: overfull_count >= 1 and not high
|
||||||
|
- low: overfull_count == 0
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections import Counter
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from datetime import date
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
PRIORITIES = ("high", "medium", "low")
|
||||||
|
PRIORITY_RANK = {name: rank for rank, name in enumerate(PRIORITIES)}
|
||||||
|
MAX_LIMIT = 500
|
||||||
|
DEFAULT_LIMIT = 50
|
||||||
|
|
||||||
|
META_KEYS = (
|
||||||
|
"source",
|
||||||
|
"generated_at",
|
||||||
|
"rows",
|
||||||
|
"containers",
|
||||||
|
"material_series",
|
||||||
|
"date_from",
|
||||||
|
"date_to",
|
||||||
|
"reference_date",
|
||||||
|
"overfull_readings",
|
||||||
|
"priority_rule",
|
||||||
|
"limitations",
|
||||||
|
)
|
||||||
|
SUMMARY_KEYS = ("priority_counts", "by_material")
|
||||||
|
MATERIAL_KEYS = (
|
||||||
|
"readings",
|
||||||
|
"overfull_count",
|
||||||
|
"last_fill",
|
||||||
|
"last_is_overfull",
|
||||||
|
"last_date",
|
||||||
|
)
|
||||||
|
READING_KEYS = ("date", "material_type", "fill_level_perc", "is_overfull")
|
||||||
|
CONTAINER_KEYS = (
|
||||||
|
"location_id",
|
||||||
|
"priority",
|
||||||
|
"priority_score",
|
||||||
|
"readings",
|
||||||
|
"overfull_count",
|
||||||
|
"overfull_rate",
|
||||||
|
"numeric_count",
|
||||||
|
"p95_fill",
|
||||||
|
"max_fill",
|
||||||
|
"mean_fill",
|
||||||
|
"last_fill",
|
||||||
|
"last_is_overfull",
|
||||||
|
"last_date",
|
||||||
|
"days_since_last",
|
||||||
|
"trend",
|
||||||
|
"materials",
|
||||||
|
"recent_readings",
|
||||||
|
)
|
||||||
|
PUBLIC_CONTAINER_KEYS = tuple(
|
||||||
|
key for key in CONTAINER_KEYS if key != "recent_readings"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PrezeroArtifactError(ValueError):
|
||||||
|
"""The PreZero artifact is missing, unreadable, or inconsistent."""
|
||||||
|
|
||||||
|
|
||||||
|
def priority_tier(overfull_count: int, overfull_rate: float) -> str:
|
||||||
|
"""Return the contract tier. ``priority_score`` is display-only."""
|
||||||
|
if overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25):
|
||||||
|
return "high"
|
||||||
|
if overfull_count >= 1:
|
||||||
|
return "medium"
|
||||||
|
return "low"
|
||||||
|
|
||||||
|
|
||||||
|
def container_sort_key(container: Mapping[str, Any]) -> tuple[int, int, float, int, str]:
|
||||||
|
"""Priority first, then overfull count/rate, staleness, and location id."""
|
||||||
|
return (
|
||||||
|
PRIORITY_RANK[container["priority"]],
|
||||||
|
-int(container["overfull_count"]),
|
||||||
|
-float(container["overfull_rate"]),
|
||||||
|
int(container["days_since_last"]),
|
||||||
|
str(container["location_id"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _fail(where: str, message: str) -> None:
|
||||||
|
raise PrezeroArtifactError(f"{where}: {message}")
|
||||||
|
|
||||||
|
|
||||||
|
def _require(mapping: Mapping[str, Any], keys: tuple[str, ...], where: str) -> None:
|
||||||
|
if not isinstance(mapping, Mapping):
|
||||||
|
_fail(where, "must be an object")
|
||||||
|
missing = [key for key in keys if key not in mapping]
|
||||||
|
if missing:
|
||||||
|
_fail(where, f"missing keys: {', '.join(missing)}")
|
||||||
|
|
||||||
|
|
||||||
|
def _integer(value: Any, where: str, *, minimum: int = 0) -> int:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int):
|
||||||
|
_fail(where, "must be an integer")
|
||||||
|
if value < minimum:
|
||||||
|
_fail(where, f"must be >= {minimum}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _number(value: Any, where: str) -> float:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||||
|
_fail(where, "must be a number")
|
||||||
|
return float(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_number(value: Any, where: str) -> float | None:
|
||||||
|
return None if value is None else _number(value, where)
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_fill(value: Any, where: str) -> int | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
fill = _integer(value, where)
|
||||||
|
if fill > 100:
|
||||||
|
_fail(where, "must be between 0 and 100")
|
||||||
|
return fill
|
||||||
|
|
||||||
|
|
||||||
|
def _date(value: Any, where: str) -> date:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
_fail(where, "must be a YYYY-MM-DD string")
|
||||||
|
try:
|
||||||
|
return date.fromisoformat(value)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise PrezeroArtifactError(f"{where}: invalid date {value!r}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_material(item: Any, where: str) -> None:
|
||||||
|
_require(item, MATERIAL_KEYS, where)
|
||||||
|
readings = _integer(item["readings"], f"{where}.readings")
|
||||||
|
overfull = _integer(item["overfull_count"], f"{where}.overfull_count")
|
||||||
|
if overfull > readings:
|
||||||
|
_fail(where, "overfull_count exceeds readings")
|
||||||
|
_optional_fill(item["last_fill"], f"{where}.last_fill")
|
||||||
|
if not isinstance(item["last_is_overfull"], bool):
|
||||||
|
_fail(f"{where}.last_is_overfull", "must be a boolean")
|
||||||
|
_date(item["last_date"], f"{where}.last_date")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_reading(item: Any, where: str) -> None:
|
||||||
|
_require(item, READING_KEYS, where)
|
||||||
|
_date(item["date"], f"{where}.date")
|
||||||
|
if not isinstance(item["material_type"], str) or not item["material_type"]:
|
||||||
|
_fail(f"{where}.material_type", "must be a non-empty string")
|
||||||
|
if not isinstance(item["fill_level_perc"], str):
|
||||||
|
_fail(f"{where}.fill_level_perc", "must be a string")
|
||||||
|
if not isinstance(item["is_overfull"], bool):
|
||||||
|
_fail(f"{where}.is_overfull", "must be a boolean")
|
||||||
|
if item["is_overfull"] is not (item["fill_level_perc"] == "overfull"):
|
||||||
|
_fail(f"{where}.is_overfull", "does not match fill_level_perc")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_container(item: Any, where: str) -> None:
|
||||||
|
_require(item, CONTAINER_KEYS, where)
|
||||||
|
location_id = item["location_id"]
|
||||||
|
if not isinstance(location_id, str) or not location_id.strip():
|
||||||
|
_fail(f"{where}.location_id", "must be a non-empty string")
|
||||||
|
priority = item["priority"]
|
||||||
|
if priority not in PRIORITIES:
|
||||||
|
_fail(f"{where}.priority", f"must be one of {', '.join(PRIORITIES)}")
|
||||||
|
|
||||||
|
readings = _integer(item["readings"], f"{where}.readings")
|
||||||
|
overfull = _integer(item["overfull_count"], f"{where}.overfull_count")
|
||||||
|
numeric = _integer(item["numeric_count"], f"{where}.numeric_count")
|
||||||
|
if overfull > readings:
|
||||||
|
_fail(where, "overfull_count exceeds readings")
|
||||||
|
if numeric > readings:
|
||||||
|
_fail(where, "numeric_count exceeds readings")
|
||||||
|
rate = _number(item["overfull_rate"], f"{where}.overfull_rate")
|
||||||
|
expected_rate = overfull / readings if readings else 0.0
|
||||||
|
if abs(rate - expected_rate) > 1e-9:
|
||||||
|
_fail(f"{where}.overfull_rate", "does not match overfull_count / readings")
|
||||||
|
if priority_tier(overfull, rate) != priority:
|
||||||
|
_fail(f"{where}.priority", "does not match the contract tier rule")
|
||||||
|
|
||||||
|
score = _number(item["priority_score"], f"{where}.priority_score")
|
||||||
|
if abs(score - round(overfull + 2 * rate, 4)) > 1e-9:
|
||||||
|
_fail(f"{where}.priority_score", "does not match the contract score")
|
||||||
|
for key in ("p95_fill", "max_fill", "mean_fill"):
|
||||||
|
_optional_number(item[key], f"{where}.{key}")
|
||||||
|
_optional_fill(item["last_fill"], f"{where}.last_fill")
|
||||||
|
if not isinstance(item["last_is_overfull"], bool):
|
||||||
|
_fail(f"{where}.last_is_overfull", "must be a boolean")
|
||||||
|
_date(item["last_date"], f"{where}.last_date")
|
||||||
|
_integer(item["days_since_last"], f"{where}.days_since_last")
|
||||||
|
_optional_number(item["trend"], f"{where}.trend")
|
||||||
|
|
||||||
|
materials = item["materials"]
|
||||||
|
if not isinstance(materials, Mapping) or not materials:
|
||||||
|
_fail(f"{where}.materials", "must be a non-empty object")
|
||||||
|
material_readings = 0
|
||||||
|
material_overfull = 0
|
||||||
|
for material, entry in materials.items():
|
||||||
|
if not isinstance(material, str) or not material:
|
||||||
|
_fail(f"{where}.materials", "keys must be non-empty material names")
|
||||||
|
_validate_material(entry, f"{where}.materials.{material}")
|
||||||
|
material_readings += entry["readings"]
|
||||||
|
material_overfull += entry["overfull_count"]
|
||||||
|
if material_readings != readings or material_overfull != overfull:
|
||||||
|
_fail(f"{where}.materials", "per-material totals do not match the container")
|
||||||
|
|
||||||
|
recent = item["recent_readings"]
|
||||||
|
if not isinstance(recent, list):
|
||||||
|
_fail(f"{where}.recent_readings", "must be an array")
|
||||||
|
if len(recent) > 10:
|
||||||
|
_fail(f"{where}.recent_readings", "holds at most 10 rows")
|
||||||
|
for index, reading in enumerate(recent):
|
||||||
|
_validate_reading(reading, f"{where}.recent_readings[{index}]")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_meta(meta: Any, container_count: int) -> None:
|
||||||
|
_require(meta, META_KEYS, "meta")
|
||||||
|
if not isinstance(meta["source"], str) or not meta["source"]:
|
||||||
|
_fail("meta.source", "must be a non-empty string")
|
||||||
|
if not isinstance(meta["generated_at"], str) or not meta["generated_at"]:
|
||||||
|
_fail("meta.generated_at", "must be a non-empty string")
|
||||||
|
for key in ("rows", "containers", "material_series", "overfull_readings"):
|
||||||
|
_integer(meta[key], f"meta.{key}")
|
||||||
|
if meta["containers"] != container_count:
|
||||||
|
_fail("meta.containers", "does not match the container array")
|
||||||
|
date_from = _date(meta["date_from"], "meta.date_from")
|
||||||
|
date_to = _date(meta["date_to"], "meta.date_to")
|
||||||
|
reference = _date(meta["reference_date"], "meta.reference_date")
|
||||||
|
if date_from > date_to:
|
||||||
|
_fail("meta.date_from", "is after meta.date_to")
|
||||||
|
if reference != date_to:
|
||||||
|
_fail("meta.reference_date", "must equal the dataset max date")
|
||||||
|
if not isinstance(meta["priority_rule"], str) or not meta["priority_rule"]:
|
||||||
|
_fail("meta.priority_rule", "must be a non-empty string")
|
||||||
|
limitations = meta["limitations"]
|
||||||
|
if not isinstance(limitations, list) or not all(
|
||||||
|
isinstance(line, str) and line for line in limitations
|
||||||
|
):
|
||||||
|
_fail("meta.limitations", "must be an array of non-empty strings")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_summary(summary: Any, meta: Mapping[str, Any]) -> None:
|
||||||
|
_require(summary, SUMMARY_KEYS, "summary")
|
||||||
|
counts = summary["priority_counts"]
|
||||||
|
_require(counts, PRIORITIES, "summary.priority_counts")
|
||||||
|
total = 0
|
||||||
|
for priority in PRIORITIES:
|
||||||
|
total += _integer(counts[priority], f"summary.priority_counts.{priority}")
|
||||||
|
if total != meta["containers"]:
|
||||||
|
_fail("summary.priority_counts", "does not add up to meta.containers")
|
||||||
|
|
||||||
|
by_material = summary["by_material"]
|
||||||
|
if not isinstance(by_material, Mapping) or not by_material:
|
||||||
|
_fail("summary.by_material", "must be a non-empty object")
|
||||||
|
readings = 0
|
||||||
|
overfull = 0
|
||||||
|
for material, entry in by_material.items():
|
||||||
|
where = f"summary.by_material.{material}"
|
||||||
|
_require(entry, ("readings", "overfull"), where)
|
||||||
|
entry_readings = _integer(entry["readings"], f"{where}.readings")
|
||||||
|
entry_overfull = _integer(entry["overfull"], f"{where}.overfull")
|
||||||
|
if entry_overfull > entry_readings:
|
||||||
|
_fail(where, "overfull exceeds readings")
|
||||||
|
readings += entry_readings
|
||||||
|
overfull += entry_overfull
|
||||||
|
if readings != meta["rows"] or overfull != meta["overfull_readings"]:
|
||||||
|
_fail("summary.by_material", "totals do not match meta rows/overfull")
|
||||||
|
|
||||||
|
|
||||||
|
class PrezeroStore:
|
||||||
|
"""Validated, in-memory view of the committed priority artifact."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
meta: dict[str, Any],
|
||||||
|
summary: dict[str, Any],
|
||||||
|
containers: list[dict[str, Any]],
|
||||||
|
) -> None:
|
||||||
|
self.meta = meta
|
||||||
|
self.summary = summary
|
||||||
|
self.containers = tuple(sorted(containers, key=container_sort_key))
|
||||||
|
self._by_id = {container["location_id"]: container for container in self.containers}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_payload(cls, payload: Any) -> PrezeroStore:
|
||||||
|
_require(payload, ("meta", "summary", "containers"), "artifact")
|
||||||
|
containers = payload["containers"]
|
||||||
|
if not isinstance(containers, list) or not containers:
|
||||||
|
_fail("containers", "must be a non-empty array")
|
||||||
|
for index, container in enumerate(containers):
|
||||||
|
_validate_container(container, f"containers[{index}]")
|
||||||
|
|
||||||
|
meta = payload["meta"]
|
||||||
|
_validate_meta(meta, len(containers))
|
||||||
|
_validate_summary(payload["summary"], meta)
|
||||||
|
|
||||||
|
ids = [container["location_id"] for container in containers]
|
||||||
|
duplicates = sorted(
|
||||||
|
location for location, count in Counter(ids).items() if count > 1
|
||||||
|
)
|
||||||
|
if duplicates:
|
||||||
|
_fail("containers", f"duplicate location_id values: {', '.join(duplicates)}")
|
||||||
|
|
||||||
|
actual = Counter(container["priority"] for container in containers)
|
||||||
|
for priority in PRIORITIES:
|
||||||
|
if actual[priority] != payload["summary"]["priority_counts"][priority]:
|
||||||
|
_fail(
|
||||||
|
"summary.priority_counts",
|
||||||
|
f"{priority} count does not match the container array",
|
||||||
|
)
|
||||||
|
return cls(meta=meta, summary=payload["summary"], containers=containers)
|
||||||
|
|
||||||
|
def summary_payload(self) -> dict[str, Any]:
|
||||||
|
return {"meta": self.meta, "summary": self.summary}
|
||||||
|
|
||||||
|
def priority_payload(
|
||||||
|
self,
|
||||||
|
priority: str | None = None,
|
||||||
|
limit: int = DEFAULT_LIMIT,
|
||||||
|
offset: int = 0,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if priority is not None and priority not in PRIORITIES:
|
||||||
|
raise ValueError(f"priority must be one of {', '.join(PRIORITIES)}")
|
||||||
|
if (
|
||||||
|
isinstance(limit, bool)
|
||||||
|
or not isinstance(limit, int)
|
||||||
|
or not 1 <= limit <= MAX_LIMIT
|
||||||
|
):
|
||||||
|
raise ValueError(f"limit must be between 1 and {MAX_LIMIT}")
|
||||||
|
if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
|
||||||
|
raise ValueError("offset must be >= 0")
|
||||||
|
|
||||||
|
matching = [
|
||||||
|
container
|
||||||
|
for container in self.containers
|
||||||
|
if priority is None or container["priority"] == priority
|
||||||
|
]
|
||||||
|
page = matching[offset : offset + limit]
|
||||||
|
return {
|
||||||
|
"meta": self.meta,
|
||||||
|
"total": len(matching),
|
||||||
|
"count": len(page),
|
||||||
|
"offset": offset,
|
||||||
|
"limit": limit,
|
||||||
|
"containers": [
|
||||||
|
{key: container[key] for key in PUBLIC_CONTAINER_KEYS}
|
||||||
|
for container in page
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
def container_payload(self, location_id: str) -> dict[str, Any] | None:
|
||||||
|
container = self._by_id.get(location_id)
|
||||||
|
if container is None:
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
"container": {key: container[key] for key in PUBLIC_CONTAINER_KEYS},
|
||||||
|
"readings": container["recent_readings"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def load_prezero(path: Path) -> PrezeroStore:
|
||||||
|
"""Read and validate the artifact. Raises FileNotFoundError when absent."""
|
||||||
|
try:
|
||||||
|
raw = path.read_text(encoding="utf-8")
|
||||||
|
except FileNotFoundError:
|
||||||
|
raise
|
||||||
|
except (OSError, UnicodeDecodeError) as exc:
|
||||||
|
raise PrezeroArtifactError(f"cannot read {path}: {exc}") from exc
|
||||||
|
try:
|
||||||
|
payload = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise PrezeroArtifactError(f"invalid JSON in {path}: {exc}") from exc
|
||||||
|
return PrezeroStore.from_payload(payload)
|
||||||
@@ -29,6 +29,7 @@ from pydantic import BaseModel, ConfigDict
|
|||||||
from . import config
|
from . import config
|
||||||
from .features import future_calendar
|
from .features import future_calendar
|
||||||
from .model import event_multiplier, future_frame, points, predict
|
from .model import event_multiplier, future_frame, points, predict
|
||||||
|
from .prezero import PrezeroStore, load_prezero
|
||||||
from .retrain import train_forecast_from_csv
|
from .retrain import train_forecast_from_csv
|
||||||
from .small import train_small_from_csv
|
from .small import train_small_from_csv
|
||||||
|
|
||||||
@@ -54,6 +55,7 @@ class Runtime:
|
|||||||
context: dict | None = None
|
context: dict | None = None
|
||||||
small_bundle: dict | None = None
|
small_bundle: dict | None = None
|
||||||
small_metrics: dict | None = None
|
small_metrics: dict | None = None
|
||||||
|
prezero: PrezeroStore | None = None
|
||||||
model_source: str = "baked"
|
model_source: str = "baked"
|
||||||
jobs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
jobs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||||
jobs_lock: threading.Lock = field(default_factory=threading.Lock)
|
jobs_lock: threading.Lock = field(default_factory=threading.Lock)
|
||||||
@@ -117,6 +119,24 @@ def _load_runtime(runtime: Runtime) -> None:
|
|||||||
if small_metrics_path.exists():
|
if small_metrics_path.exists():
|
||||||
runtime.small_metrics = json.loads(small_metrics_path.read_text())
|
runtime.small_metrics = json.loads(small_metrics_path.read_text())
|
||||||
|
|
||||||
|
# PreZero is an optional proof surface: a missing or malformed artifact
|
||||||
|
# must disable only /prezero/*, never the forecast/training routes.
|
||||||
|
prezero_path = config.prezero_artifact()
|
||||||
|
try:
|
||||||
|
runtime.prezero = load_prezero(prezero_path)
|
||||||
|
log.info(
|
||||||
|
"loaded prezero artifact with %d containers",
|
||||||
|
len(runtime.prezero.containers),
|
||||||
|
)
|
||||||
|
except FileNotFoundError:
|
||||||
|
log.warning("prezero artifact missing: %s; /prezero routes will return 503", prezero_path)
|
||||||
|
except Exception as exc:
|
||||||
|
log.error(
|
||||||
|
"prezero artifact invalid (%s); /prezero routes will return 503: %s",
|
||||||
|
prezero_path,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- schemas
|
# --------------------------------------------------------------------------- schemas
|
||||||
|
|
||||||
@@ -153,6 +173,12 @@ class SimulateRequest(BaseModel):
|
|||||||
# --------------------------------------------------------------------------- helpers
|
# --------------------------------------------------------------------------- helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _prezero(runtime: Runtime) -> PrezeroStore:
|
||||||
|
if runtime.prezero is None:
|
||||||
|
raise HTTPException(503, "PreZero priority artifact is not loaded")
|
||||||
|
return runtime.prezero
|
||||||
|
|
||||||
|
|
||||||
def _require_token(authorization: str | None) -> None:
|
def _require_token(authorization: str | None) -> None:
|
||||||
expected = config.control_token()
|
expected = config.control_token()
|
||||||
if not expected:
|
if not expected:
|
||||||
@@ -536,6 +562,31 @@ def create_app() -> FastAPI:
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@app.get("/prezero/summary")
|
||||||
|
def prezero_summary() -> dict:
|
||||||
|
"""Priority counts and material totals for the PreZero proof."""
|
||||||
|
return _prezero(runtime()).summary_payload()
|
||||||
|
|
||||||
|
@app.get("/prezero/priority")
|
||||||
|
def prezero_priority(
|
||||||
|
priority: str | None = None,
|
||||||
|
limit: int = 50,
|
||||||
|
offset: int = 0,
|
||||||
|
) -> dict:
|
||||||
|
"""Ranked container locations, optionally filtered by priority tier."""
|
||||||
|
try:
|
||||||
|
return _prezero(runtime()).priority_payload(priority, limit, offset)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(400, str(exc)) from exc
|
||||||
|
|
||||||
|
@app.get("/prezero/containers/{location_id}")
|
||||||
|
def prezero_container(location_id: str) -> dict:
|
||||||
|
"""One container location plus its most recent 10 readings."""
|
||||||
|
payload = _prezero(runtime()).container_payload(location_id)
|
||||||
|
if payload is None:
|
||||||
|
raise HTTPException(404, f"unknown location: {location_id}")
|
||||||
|
return payload
|
||||||
|
|
||||||
@app.post("/predict")
|
@app.post("/predict")
|
||||||
def predict_endpoint(body: PredictRequest) -> dict:
|
def predict_endpoint(body: PredictRequest) -> dict:
|
||||||
state = runtime()
|
state = runtime()
|
||||||
|
|||||||
@@ -0,0 +1,335 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import json
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from pulsecast import config
|
||||||
|
from pulsecast.prezero import (
|
||||||
|
CONTAINER_KEYS,
|
||||||
|
MATERIAL_KEYS,
|
||||||
|
META_KEYS,
|
||||||
|
PRIORITIES,
|
||||||
|
READING_KEYS,
|
||||||
|
PrezeroArtifactError,
|
||||||
|
PrezeroStore,
|
||||||
|
priority_tier,
|
||||||
|
)
|
||||||
|
|
||||||
|
EXPECTED = {
|
||||||
|
"rows": 11_701,
|
||||||
|
"containers": 449,
|
||||||
|
"material_series": 1_338,
|
||||||
|
"overfull_readings": 1_192,
|
||||||
|
"high": 88,
|
||||||
|
"medium": 248,
|
||||||
|
"low": 113,
|
||||||
|
"date_from": "2025-10-17",
|
||||||
|
"date_to": "2026-09-30",
|
||||||
|
}
|
||||||
|
PUBLIC_KEYS = tuple(key for key in CONTAINER_KEYS if key != "recent_readings")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def artifact() -> dict:
|
||||||
|
return json.loads(config.prezero_artifact().read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def store(artifact) -> PrezeroStore:
|
||||||
|
return PrezeroStore.from_payload(artifact)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def summary(client) -> dict:
|
||||||
|
response = client.get("/prezero/summary")
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
return response.json()
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- rule
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("overfull_count", "overfull_rate", "expected"),
|
||||||
|
[
|
||||||
|
(0, 0.0, "low"),
|
||||||
|
(1, 0.01, "medium"),
|
||||||
|
(4, 0.99, "high"),
|
||||||
|
(2, 0.25, "high"),
|
||||||
|
(2, 0.2499, "medium"),
|
||||||
|
(5, 0.0, "high"),
|
||||||
|
(1, 1.0, "medium"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_priority_rule(overfull_count, overfull_rate, expected):
|
||||||
|
assert priority_tier(overfull_count, overfull_rate) == expected
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- summary
|
||||||
|
|
||||||
|
|
||||||
|
def test_summary_counts(summary):
|
||||||
|
assert set(summary) == {"meta", "summary"}
|
||||||
|
meta = summary["meta"]
|
||||||
|
for key in META_KEYS:
|
||||||
|
assert key in meta, key
|
||||||
|
assert meta["rows"] == EXPECTED["rows"]
|
||||||
|
assert meta["containers"] == EXPECTED["containers"]
|
||||||
|
assert meta["material_series"] == EXPECTED["material_series"]
|
||||||
|
assert meta["overfull_readings"] == EXPECTED["overfull_readings"]
|
||||||
|
assert meta["date_from"] == EXPECTED["date_from"]
|
||||||
|
assert meta["date_to"] == EXPECTED["date_to"]
|
||||||
|
assert meta["reference_date"] == EXPECTED["date_to"]
|
||||||
|
assert meta["source"] == "glass_fill_levels_anonymized_f.csv"
|
||||||
|
assert "overfull_count >= 5" in meta["priority_rule"]
|
||||||
|
assert any("not a route" in line for line in meta["limitations"])
|
||||||
|
|
||||||
|
counts = summary["summary"]["priority_counts"]
|
||||||
|
assert counts == {
|
||||||
|
"high": EXPECTED["high"],
|
||||||
|
"medium": EXPECTED["medium"],
|
||||||
|
"low": EXPECTED["low"],
|
||||||
|
}
|
||||||
|
assert summary["summary"]["by_material"] == {
|
||||||
|
"white": {"readings": 4_432, "overfull": 731},
|
||||||
|
"green": {"readings": 4_169, "overfull": 357},
|
||||||
|
"brown": {"readings": 3_100, "overfull": 104},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- priority
|
||||||
|
|
||||||
|
|
||||||
|
def test_priority_defaults_and_filter_totals(client, summary):
|
||||||
|
payload = client.get("/prezero/priority").json()
|
||||||
|
assert set(payload) == {"meta", "total", "count", "offset", "limit", "containers"}
|
||||||
|
assert payload["meta"] == summary["meta"]
|
||||||
|
assert payload["total"] == EXPECTED["containers"]
|
||||||
|
assert payload["limit"] == 50
|
||||||
|
assert payload["offset"] == 0
|
||||||
|
assert payload["count"] == 50
|
||||||
|
assert all(container["priority"] == "high" for container in payload["containers"])
|
||||||
|
for priority in PRIORITIES:
|
||||||
|
filtered = client.get("/prezero/priority", params={"priority": priority}).json()
|
||||||
|
assert filtered["total"] == EXPECTED[priority]
|
||||||
|
assert filtered["count"] == min(50, EXPECTED[priority])
|
||||||
|
assert all(row["priority"] == priority for row in filtered["containers"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_priority_limit_offset_and_count(client, store, artifact):
|
||||||
|
payload = client.get(
|
||||||
|
"/prezero/priority", params={"priority": "high", "limit": 3}
|
||||||
|
).json()
|
||||||
|
assert payload["total"] == EXPECTED["high"]
|
||||||
|
assert payload["count"] == 3
|
||||||
|
high_ids = [
|
||||||
|
row["location_id"]
|
||||||
|
for row in artifact["containers"]
|
||||||
|
if row["priority"] == "high"
|
||||||
|
]
|
||||||
|
expected_ids = high_ids[:3]
|
||||||
|
assert [row["location_id"] for row in payload["containers"]] == expected_ids
|
||||||
|
for row in payload["containers"]:
|
||||||
|
assert set(row) == set(PUBLIC_KEYS)
|
||||||
|
assert "recent_readings" not in row
|
||||||
|
|
||||||
|
paged = client.get(
|
||||||
|
"/prezero/priority",
|
||||||
|
params={"priority": "high", "limit": 2, "offset": 2},
|
||||||
|
).json()
|
||||||
|
assert paged["total"] == EXPECTED["high"]
|
||||||
|
assert paged["count"] == 2
|
||||||
|
assert paged["offset"] == 2
|
||||||
|
assert [row["location_id"] for row in paged["containers"]] == high_ids[2:4]
|
||||||
|
|
||||||
|
beyond = client.get(
|
||||||
|
"/prezero/priority", params={"priority": "high", "limit": 10, "offset": 88}
|
||||||
|
).json()
|
||||||
|
assert beyond["total"] == EXPECTED["high"]
|
||||||
|
assert beyond["count"] == 0
|
||||||
|
assert beyond["containers"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_priority_is_sorted_by_contract(client, store):
|
||||||
|
payload = client.get("/prezero/priority", params={"limit": 500}).json()
|
||||||
|
assert payload["total"] == EXPECTED["containers"]
|
||||||
|
assert payload["count"] == EXPECTED["containers"]
|
||||||
|
rows = payload["containers"]
|
||||||
|
assert [row["location_id"] for row in rows] == [
|
||||||
|
container["location_id"] for container in store.containers
|
||||||
|
]
|
||||||
|
rank = {priority: index for index, priority in enumerate(PRIORITIES)}
|
||||||
|
keys = [
|
||||||
|
(
|
||||||
|
rank[row["priority"]],
|
||||||
|
-row["overfull_count"],
|
||||||
|
-row["overfull_rate"],
|
||||||
|
row["days_since_last"],
|
||||||
|
row["location_id"],
|
||||||
|
)
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
assert keys == sorted(keys)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- container
|
||||||
|
|
||||||
|
|
||||||
|
def test_known_container_and_recent_readings(client, artifact):
|
||||||
|
known = next(row for row in artifact["containers"] if row["priority"] == "high")
|
||||||
|
location_id = known["location_id"]
|
||||||
|
response = client.get(f"/prezero/containers/{location_id}")
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
payload = response.json()
|
||||||
|
|
||||||
|
assert set(payload) == {"container", "readings"}
|
||||||
|
container = payload["container"]
|
||||||
|
assert set(container) == set(PUBLIC_KEYS)
|
||||||
|
for key in PUBLIC_KEYS:
|
||||||
|
assert container[key] == known[key]
|
||||||
|
assert len(payload["readings"]) == min(10, known["readings"])
|
||||||
|
dates = [reading["date"] for reading in payload["readings"]]
|
||||||
|
assert dates == sorted(dates, reverse=True)
|
||||||
|
for reading in payload["readings"]:
|
||||||
|
assert set(reading) == set(READING_KEYS)
|
||||||
|
assert isinstance(reading["fill_level_perc"], str)
|
||||||
|
assert reading["is_overfull"] is (reading["fill_level_perc"] == "overfull")
|
||||||
|
|
||||||
|
|
||||||
|
def test_sparse_container_uses_null_numeric_fields(client, artifact):
|
||||||
|
sparse = next(row for row in artifact["containers"] if row["numeric_count"] == 0)
|
||||||
|
payload = client.get(f"/prezero/containers/{sparse['location_id']}").json()
|
||||||
|
assert payload["container"]["p95_fill"] is None
|
||||||
|
assert payload["container"]["max_fill"] is None
|
||||||
|
assert payload["container"]["mean_fill"] is None
|
||||||
|
assert payload["container"]["trend"] is None
|
||||||
|
assert payload["container"]["last_is_overfull"] is True
|
||||||
|
assert payload["container"]["last_fill"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_container_404(client):
|
||||||
|
response = client.get("/prezero/containers/does-not-exist")
|
||||||
|
assert response.status_code == 404
|
||||||
|
assert response.json() == {
|
||||||
|
"error": {
|
||||||
|
"code": "not_found",
|
||||||
|
"message": "unknown location: does-not-exist",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("params", "message"),
|
||||||
|
[
|
||||||
|
({"priority": "urgent"}, "priority must be one of high, medium, low"),
|
||||||
|
({"priority": "HIGH"}, "priority must be one of high, medium, low"),
|
||||||
|
({"limit": 0}, "limit must be between 1 and 500"),
|
||||||
|
({"limit": 501}, "limit must be between 1 and 500"),
|
||||||
|
({"offset": -1}, "offset must be >= 0"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_invalid_queries_400(client, params, message):
|
||||||
|
response = client.get("/prezero/priority", params=params)
|
||||||
|
assert response.status_code == 400
|
||||||
|
assert response.json() == {"error": {"code": "bad_request", "message": message}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_integer_query_400(client):
|
||||||
|
response = client.get("/prezero/priority", params={"limit": "many"})
|
||||||
|
assert response.status_code == 400
|
||||||
|
assert response.json()["error"]["code"] == "bad_request"
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- artifact
|
||||||
|
|
||||||
|
|
||||||
|
def test_artifact_schema(artifact, store):
|
||||||
|
assert set(artifact) == {"meta", "summary", "containers"}
|
||||||
|
meta = artifact["meta"]
|
||||||
|
assert meta["rows"] == EXPECTED["rows"]
|
||||||
|
assert meta["containers"] == EXPECTED["containers"]
|
||||||
|
assert meta["material_series"] == EXPECTED["material_series"]
|
||||||
|
assert meta["overfull_readings"] == EXPECTED["overfull_readings"]
|
||||||
|
assert meta["date_from"] == EXPECTED["date_from"]
|
||||||
|
assert meta["date_to"] == EXPECTED["date_to"]
|
||||||
|
assert meta["reference_date"] == EXPECTED["date_to"]
|
||||||
|
|
||||||
|
containers = artifact["containers"]
|
||||||
|
assert len(containers) == EXPECTED["containers"]
|
||||||
|
assert len(store.containers) == EXPECTED["containers"]
|
||||||
|
assert len({row["location_id"] for row in containers}) == EXPECTED["containers"]
|
||||||
|
assert Counter(row["priority"] for row in containers) == {
|
||||||
|
"high": EXPECTED["high"],
|
||||||
|
"medium": EXPECTED["medium"],
|
||||||
|
"low": EXPECTED["low"],
|
||||||
|
}
|
||||||
|
|
||||||
|
for row in containers:
|
||||||
|
assert set(row) == set(CONTAINER_KEYS)
|
||||||
|
assert row["last_date"].count("-") == 2
|
||||||
|
assert len(row["recent_readings"]) == min(10, row["readings"])
|
||||||
|
assert row["readings"] == sum(
|
||||||
|
material["readings"] for material in row["materials"].values()
|
||||||
|
)
|
||||||
|
assert row["overfull_count"] == sum(
|
||||||
|
material["overfull_count"] for material in row["materials"].values()
|
||||||
|
)
|
||||||
|
for material in row["materials"].values():
|
||||||
|
assert set(material) == set(MATERIAL_KEYS)
|
||||||
|
|
||||||
|
material_totals = Counter()
|
||||||
|
material_overfull = Counter()
|
||||||
|
for row in containers:
|
||||||
|
for name, material in row["materials"].items():
|
||||||
|
material_totals[name] += material["readings"]
|
||||||
|
material_overfull[name] += material["overfull_count"]
|
||||||
|
assert material_totals == {
|
||||||
|
"white": 4_432,
|
||||||
|
"green": 4_169,
|
||||||
|
"brown": 3_100,
|
||||||
|
}
|
||||||
|
assert material_overfull == {"white": 731, "green": 357, "brown": 104}
|
||||||
|
assert sum(row["readings"] for row in containers) == EXPECTED["rows"]
|
||||||
|
assert sum(row["overfull_count"] for row in containers) == EXPECTED[
|
||||||
|
"overfull_readings"
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_artifact_rejects_inconsistent_tier(artifact):
|
||||||
|
payload = copy.deepcopy(artifact)
|
||||||
|
payload["containers"][0]["priority"] = "urgent"
|
||||||
|
with pytest.raises(PrezeroArtifactError, match="priority"):
|
||||||
|
PrezeroStore.from_payload(payload)
|
||||||
|
|
||||||
|
|
||||||
|
def test_artifact_rejects_wrong_summary_count(artifact):
|
||||||
|
payload = copy.deepcopy(artifact)
|
||||||
|
payload["summary"]["priority_counts"]["high"] += 1
|
||||||
|
with pytest.raises(PrezeroArtifactError, match="priority_counts"):
|
||||||
|
PrezeroStore.from_payload(payload)
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_artifact_returns_503(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setenv(
|
||||||
|
"PULSECAST_PREZERO_ARTIFACT", str(Path(tmp_path) / "missing.json")
|
||||||
|
)
|
||||||
|
from pulsecast.server import create_app
|
||||||
|
|
||||||
|
with TestClient(create_app()) as isolated:
|
||||||
|
assert isolated.get("/healthz").status_code == 200
|
||||||
|
assert isolated.get("/readyz").status_code == 200
|
||||||
|
assert isolated.get("/meta").status_code == 200
|
||||||
|
assert isolated.get("/bootstrap").status_code == 200
|
||||||
|
for path in (
|
||||||
|
"/prezero/summary",
|
||||||
|
"/prezero/priority",
|
||||||
|
"/prezero/containers/anything",
|
||||||
|
):
|
||||||
|
response = isolated.get(path)
|
||||||
|
assert response.status_code == 503, path
|
||||||
|
assert response.json()["error"]["code"] == "unavailable"
|
||||||
Reference in new issue
Block a user