Files
ML-py/scripts/build_prezero.py
T
Noel Sabia ad63d1caf3
ci / check (push) Successful in 30s
ci / check (pull_request) Successful in 29s
feat: prezero priority snapshot and internal API
Build artifacts/prezero_priority.json from the anonymized glass fill-level export (11,701 rows, 449 containers, 1,338 location/material series). The contract tier rule yields high 88, medium 248, low 113 with 1,192 overfull readings.

Add /prezero/summary, /prezero/priority, and /prezero/containers/{id} to the internal FastAPI service, with startup validation, filtering/pagination, recent reading history, and 503 isolation when the artifact is missing. Raw CSV stays gitignored; the derived 1.1 MB artifact is the source of truth. This is a priority proof, not a route: the dataset has no coordinates.
2026-11-14 18:06:02 +01:00

330 lines
11 KiB
Python

#!/usr/bin/env python3
"""Build ``artifacts/prezero_priority.json`` from the raw fill-level export.
The raw CSV is challenge data and stays git-ignored. The derived artifact is
committed and is the source of truth for the internal ``/prezero`` routes.
Priority rule (contract, transparent, no invented labels):
high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25)
medium: overfull_count >= 1 and not high
low: overfull_count == 0
There are no coordinates in the dataset: this is a service-priority proof, not
a route. ``priority_score`` is a sorting/display convenience; the tier is the
contract.
Examples:
uv run python scripts/build_prezero.py
uv run python scripts/build_prezero.py --csv data/raw/glass_fill_levels_anonymized_f.csv
"""
from __future__ import annotations
import argparse
import csv
import json
import sys
from collections import Counter, defaultdict
from datetime import UTC, date, datetime
from pathlib import Path
from typing import Any
import numpy as np
from pulsecast import config
from pulsecast.prezero import container_sort_key, priority_tier
REQUIRED_COLUMNS = ("Location_id", "material_type", "date", "fill_level_perc")
MATERIAL_ORDER = ("white", "green", "brown")
PRIORITY_RULE = (
"high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25); "
"medium: overfull_count >= 1 and not high; low: 0"
)
LIMITATIONS = [
"No coordinates in the dataset; this is a priority proof, not a route.",
"No container capacity or depot data.",
"Overfull is a reported state, not a measured volume.",
]
class BuildError(ValueError):
"""The input CSV does not satisfy the PreZero contract."""
def _normalized_columns(fieldnames: list[str] | None, path: Path) -> dict[str, str]:
"""Map BOM/whitespace-normalized names back to the CSV's raw names."""
if not fieldnames:
raise BuildError(f"{path}: missing header row")
mapping: dict[str, str] = {}
for raw in fieldnames:
name = str(raw).strip().lstrip("\ufeff")
if not name:
raise BuildError(f"{path}: blank column name")
if name in mapping:
raise BuildError(f"{path}: duplicate column after trimming: {name}")
mapping[name] = raw
missing = [name for name in REQUIRED_COLUMNS if name not in mapping]
if missing:
raise BuildError(
f"{path}: missing required columns: {', '.join(missing)}; "
f"found: {', '.join(mapping)}"
)
return mapping
def _read_rows(path: Path) -> list[dict[str, Any]]:
"""Read the export as strings, normalize headers/cells, validate every row."""
try:
handle = path.open("r", encoding="utf-8-sig", newline="")
except FileNotFoundError as exc:
raise BuildError(
f"CSV not found: {path}. Copy the raw export there or pass --csv."
) from exc
except OSError as exc:
raise BuildError(f"cannot read {path}: {exc}") from exc
rows: list[dict[str, Any]] = []
with handle:
reader = csv.DictReader(handle)
columns = _normalized_columns(reader.fieldnames, path)
for line_number, raw_row in enumerate(reader, start=2):
row = {
name: str(raw_row.get(source) or "").strip()
for name, source in columns.items()
}
blank = [name for name in REQUIRED_COLUMNS if not row[name]]
if blank:
raise BuildError(f"{path}:{line_number}: blank values in {', '.join(blank)}")
try:
reading_date = date.fromisoformat(row["date"])
except ValueError as exc:
raise BuildError(
f"{path}:{line_number}: invalid date {row['date']!r}"
) from exc
fill_text = row["fill_level_perc"]
if fill_text == "overfull":
fill: int | None = None
is_overfull = True
else:
try:
fill = int(fill_text)
except ValueError as exc:
raise BuildError(
f"{path}:{line_number}: fill_level_perc must be an integer "
f"0-100 or 'overfull', got {fill_text!r}"
) from exc
if not 0 <= fill <= 100:
raise BuildError(
f"{path}:{line_number}: fill_level_perc out of range: {fill}"
)
is_overfull = False
rows.append(
{
"index": len(rows),
"location_id": row["Location_id"],
"material_type": row["material_type"],
"date": row["date"],
"date_value": reading_date,
"fill_level_perc": fill_text,
"fill": fill,
"is_overfull": is_overfull,
}
)
if not rows:
raise BuildError(f"{path}: no data rows")
return rows
def _ordered(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Chronological; input order breaks ties on the same date."""
return sorted(rows, key=lambda row: (row["date_value"], row["index"]))
def _trend(numeric: list[int]) -> float | None:
"""Least-squares slope over the last up to 3 numeric readings (points/read)."""
if len(numeric) < 2:
return None
tail = numeric[-3:]
x_mean = (len(tail) - 1) / 2
y_mean = sum(tail) / len(tail)
numerator = sum((x - x_mean) * (y - y_mean) for x, y in enumerate(tail))
denominator = sum((x - x_mean) ** 2 for x in range(len(tail)))
slope = numerator / denominator
return round(slope, 2) if slope else 0.0
def _material_names(materials: set[str]) -> list[str]:
known = [name for name in MATERIAL_ORDER if name in materials]
return known + sorted(materials - set(known))
def _material_summary(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
ordered = _ordered(rows)
last = ordered[-1]
return {
"readings": len(ordered),
"overfull_count": sum(row["is_overfull"] for row in ordered),
"last_fill": last["fill"],
"last_is_overfull": last["is_overfull"],
"last_date": last["date"],
}
def _container(rows: list[dict[str, Any]], reference: date) -> dict[str, Any]:
ordered = _ordered(rows)
readings = len(ordered)
overfull_count = sum(row["is_overfull"] for row in ordered)
overfull_rate = overfull_count / readings
numeric = [row["fill"] for row in ordered if row["fill"] is not None]
last = ordered[-1]
by_material: dict[str, list[dict[str, Any]]] = defaultdict(list)
for row in ordered:
by_material[row["material_type"]].append(row)
materials = {
name: _material_summary(by_material[name])
for name in _material_names(set(by_material))
}
p95_fill = round(float(np.percentile(numeric, 95)), 2) if numeric else None
mean_fill = round(sum(numeric) / len(numeric), 2) if numeric else None
recent = [
{
"date": row["date"],
"material_type": row["material_type"],
"fill_level_perc": row["fill_level_perc"],
"is_overfull": row["is_overfull"],
}
for row in reversed(ordered[-10:])
]
return {
"location_id": ordered[0]["location_id"],
"priority": priority_tier(overfull_count, overfull_rate),
"priority_score": round(overfull_count + 2 * overfull_rate, 4),
"readings": readings,
"overfull_count": overfull_count,
"overfull_rate": overfull_rate,
"numeric_count": len(numeric),
"p95_fill": p95_fill,
"max_fill": max(numeric) if numeric else None,
"mean_fill": mean_fill,
"last_fill": last["fill"],
"last_is_overfull": last["is_overfull"],
"last_date": last["date"],
"days_since_last": (reference - last["date_value"]).days,
"trend": _trend(numeric),
"materials": materials,
"recent_readings": recent,
}
def build(csv_path: Path) -> dict[str, Any]:
rows = _read_rows(csv_path)
reference = max(row["date_value"] for row in rows)
date_from = min(row["date_value"] for row in rows)
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
for row in rows:
grouped[row["location_id"]].append(row)
containers = [
_container(grouped[location_id], reference)
for location_id in sorted(grouped)
]
containers.sort(key=container_sort_key)
material_counts = Counter(row["material_type"] for row in rows)
material_overfull = Counter(
row["material_type"] for row in rows if row["is_overfull"]
)
priority_counts = Counter(container["priority"] for container in containers)
by_material = {
name: {
"readings": material_counts[name],
"overfull": material_overfull[name],
}
for name in _material_names(set(material_counts))
}
meta = {
"source": csv_path.name,
"generated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
"rows": len(rows),
"containers": len(containers),
"material_series": len(
{(row["location_id"], row["material_type"]) for row in rows}
),
"date_from": date_from.isoformat(),
"date_to": reference.isoformat(),
"reference_date": reference.isoformat(),
"overfull_readings": sum(row["is_overfull"] for row in rows),
"priority_rule": PRIORITY_RULE,
"limitations": list(LIMITATIONS),
}
summary = {
"priority_counts": {
"high": priority_counts["high"],
"medium": priority_counts["medium"],
"low": priority_counts["low"],
},
"by_material": by_material,
}
return {"meta": meta, "summary": summary, "containers": containers}
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--csv",
type=Path,
default=config.prezero_csv(),
help="raw fill-level CSV (git-ignored; default: %(default)s)",
)
parser.add_argument(
"--out",
type=Path,
default=config.prezero_artifact(),
help="artifact destination (default: %(default)s)",
)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = _parse_args(argv)
try:
payload = build(args.csv)
except BuildError as exc:
print(f"build_prezero: {exc}", file=sys.stderr)
return 1
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(
json.dumps(payload, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
meta = payload["meta"]
counts = payload["summary"]["priority_counts"]
print(f"wrote {args.out} ({args.out.stat().st_size:,} bytes)")
print(
f"rows={meta['rows']} containers={meta['containers']} "
f"material_series={meta['material_series']} "
f"overfull={meta['overfull_readings']}"
)
print(
f"date_range={meta['date_from']}..{meta['date_to']} "
f"high={counts['high']} medium={counts['medium']} low={counts['low']}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())