feat: prezero priority snapshot and internal API
Build artifacts/prezero_priority.json from the anonymized glass fill-level export (11,701 rows, 449 containers, 1,338 location/material series). The contract tier rule yields high 88, medium 248, low 113 with 1,192 overfull readings.
Add /prezero/summary, /prezero/priority, and /prezero/containers/{id} to the internal FastAPI service, with startup validation, filtering/pagination, recent reading history, and 503 isolation when the artifact is missing. Raw CSV stays gitignored; the derived 1.1 MB artifact is the source of truth. This is a priority proof, not a route: the dataset has no coordinates.
This commit is contained in:
1 parent
507f33b7ea
commit
ad63d1caf3
9 files changed
+46198
-4
No files matched your search
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build ``artifacts/prezero_priority.json`` from the raw fill-level export.
|
||||
|
||||
The raw CSV is challenge data and stays git-ignored. The derived artifact is
|
||||
committed and is the source of truth for the internal ``/prezero`` routes.
|
||||
|
||||
Priority rule (contract, transparent, no invented labels):
|
||||
|
||||
high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25)
|
||||
medium: overfull_count >= 1 and not high
|
||||
low: overfull_count == 0
|
||||
|
||||
There are no coordinates in the dataset: this is a service-priority proof, not
|
||||
a route. ``priority_score`` is a sorting/display convenience; the tier is the
|
||||
contract.
|
||||
|
||||
Examples:
|
||||
uv run python scripts/build_prezero.py
|
||||
uv run python scripts/build_prezero.py --csv data/raw/glass_fill_levels_anonymized_f.csv
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import UTC, date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from pulsecast import config
|
||||
from pulsecast.prezero import container_sort_key, priority_tier
|
||||
|
||||
REQUIRED_COLUMNS = ("Location_id", "material_type", "date", "fill_level_perc")
|
||||
MATERIAL_ORDER = ("white", "green", "brown")
|
||||
PRIORITY_RULE = (
|
||||
"high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25); "
|
||||
"medium: overfull_count >= 1 and not high; low: 0"
|
||||
)
|
||||
LIMITATIONS = [
|
||||
"No coordinates in the dataset; this is a priority proof, not a route.",
|
||||
"No container capacity or depot data.",
|
||||
"Overfull is a reported state, not a measured volume.",
|
||||
]
|
||||
|
||||
|
||||
class BuildError(ValueError):
|
||||
"""The input CSV does not satisfy the PreZero contract."""
|
||||
|
||||
|
||||
def _normalized_columns(fieldnames: list[str] | None, path: Path) -> dict[str, str]:
|
||||
"""Map BOM/whitespace-normalized names back to the CSV's raw names."""
|
||||
if not fieldnames:
|
||||
raise BuildError(f"{path}: missing header row")
|
||||
mapping: dict[str, str] = {}
|
||||
for raw in fieldnames:
|
||||
name = str(raw).strip().lstrip("\ufeff")
|
||||
if not name:
|
||||
raise BuildError(f"{path}: blank column name")
|
||||
if name in mapping:
|
||||
raise BuildError(f"{path}: duplicate column after trimming: {name}")
|
||||
mapping[name] = raw
|
||||
missing = [name for name in REQUIRED_COLUMNS if name not in mapping]
|
||||
if missing:
|
||||
raise BuildError(
|
||||
f"{path}: missing required columns: {', '.join(missing)}; "
|
||||
f"found: {', '.join(mapping)}"
|
||||
)
|
||||
return mapping
|
||||
|
||||
|
||||
def _read_rows(path: Path) -> list[dict[str, Any]]:
|
||||
"""Read the export as strings, normalize headers/cells, validate every row."""
|
||||
try:
|
||||
handle = path.open("r", encoding="utf-8-sig", newline="")
|
||||
except FileNotFoundError as exc:
|
||||
raise BuildError(
|
||||
f"CSV not found: {path}. Copy the raw export there or pass --csv."
|
||||
) from exc
|
||||
except OSError as exc:
|
||||
raise BuildError(f"cannot read {path}: {exc}") from exc
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
with handle:
|
||||
reader = csv.DictReader(handle)
|
||||
columns = _normalized_columns(reader.fieldnames, path)
|
||||
for line_number, raw_row in enumerate(reader, start=2):
|
||||
row = {
|
||||
name: str(raw_row.get(source) or "").strip()
|
||||
for name, source in columns.items()
|
||||
}
|
||||
blank = [name for name in REQUIRED_COLUMNS if not row[name]]
|
||||
if blank:
|
||||
raise BuildError(f"{path}:{line_number}: blank values in {', '.join(blank)}")
|
||||
|
||||
try:
|
||||
reading_date = date.fromisoformat(row["date"])
|
||||
except ValueError as exc:
|
||||
raise BuildError(
|
||||
f"{path}:{line_number}: invalid date {row['date']!r}"
|
||||
) from exc
|
||||
|
||||
fill_text = row["fill_level_perc"]
|
||||
if fill_text == "overfull":
|
||||
fill: int | None = None
|
||||
is_overfull = True
|
||||
else:
|
||||
try:
|
||||
fill = int(fill_text)
|
||||
except ValueError as exc:
|
||||
raise BuildError(
|
||||
f"{path}:{line_number}: fill_level_perc must be an integer "
|
||||
f"0-100 or 'overfull', got {fill_text!r}"
|
||||
) from exc
|
||||
if not 0 <= fill <= 100:
|
||||
raise BuildError(
|
||||
f"{path}:{line_number}: fill_level_perc out of range: {fill}"
|
||||
)
|
||||
is_overfull = False
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"index": len(rows),
|
||||
"location_id": row["Location_id"],
|
||||
"material_type": row["material_type"],
|
||||
"date": row["date"],
|
||||
"date_value": reading_date,
|
||||
"fill_level_perc": fill_text,
|
||||
"fill": fill,
|
||||
"is_overfull": is_overfull,
|
||||
}
|
||||
)
|
||||
if not rows:
|
||||
raise BuildError(f"{path}: no data rows")
|
||||
return rows
|
||||
|
||||
|
||||
def _ordered(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Chronological; input order breaks ties on the same date."""
|
||||
return sorted(rows, key=lambda row: (row["date_value"], row["index"]))
|
||||
|
||||
|
||||
def _trend(numeric: list[int]) -> float | None:
|
||||
"""Least-squares slope over the last up to 3 numeric readings (points/read)."""
|
||||
if len(numeric) < 2:
|
||||
return None
|
||||
tail = numeric[-3:]
|
||||
x_mean = (len(tail) - 1) / 2
|
||||
y_mean = sum(tail) / len(tail)
|
||||
numerator = sum((x - x_mean) * (y - y_mean) for x, y in enumerate(tail))
|
||||
denominator = sum((x - x_mean) ** 2 for x in range(len(tail)))
|
||||
slope = numerator / denominator
|
||||
return round(slope, 2) if slope else 0.0
|
||||
|
||||
|
||||
def _material_names(materials: set[str]) -> list[str]:
|
||||
known = [name for name in MATERIAL_ORDER if name in materials]
|
||||
return known + sorted(materials - set(known))
|
||||
|
||||
|
||||
def _material_summary(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
ordered = _ordered(rows)
|
||||
last = ordered[-1]
|
||||
return {
|
||||
"readings": len(ordered),
|
||||
"overfull_count": sum(row["is_overfull"] for row in ordered),
|
||||
"last_fill": last["fill"],
|
||||
"last_is_overfull": last["is_overfull"],
|
||||
"last_date": last["date"],
|
||||
}
|
||||
|
||||
|
||||
def _container(rows: list[dict[str, Any]], reference: date) -> dict[str, Any]:
|
||||
ordered = _ordered(rows)
|
||||
readings = len(ordered)
|
||||
overfull_count = sum(row["is_overfull"] for row in ordered)
|
||||
overfull_rate = overfull_count / readings
|
||||
numeric = [row["fill"] for row in ordered if row["fill"] is not None]
|
||||
last = ordered[-1]
|
||||
|
||||
by_material: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in ordered:
|
||||
by_material[row["material_type"]].append(row)
|
||||
materials = {
|
||||
name: _material_summary(by_material[name])
|
||||
for name in _material_names(set(by_material))
|
||||
}
|
||||
|
||||
p95_fill = round(float(np.percentile(numeric, 95)), 2) if numeric else None
|
||||
mean_fill = round(sum(numeric) / len(numeric), 2) if numeric else None
|
||||
recent = [
|
||||
{
|
||||
"date": row["date"],
|
||||
"material_type": row["material_type"],
|
||||
"fill_level_perc": row["fill_level_perc"],
|
||||
"is_overfull": row["is_overfull"],
|
||||
}
|
||||
for row in reversed(ordered[-10:])
|
||||
]
|
||||
|
||||
return {
|
||||
"location_id": ordered[0]["location_id"],
|
||||
"priority": priority_tier(overfull_count, overfull_rate),
|
||||
"priority_score": round(overfull_count + 2 * overfull_rate, 4),
|
||||
"readings": readings,
|
||||
"overfull_count": overfull_count,
|
||||
"overfull_rate": overfull_rate,
|
||||
"numeric_count": len(numeric),
|
||||
"p95_fill": p95_fill,
|
||||
"max_fill": max(numeric) if numeric else None,
|
||||
"mean_fill": mean_fill,
|
||||
"last_fill": last["fill"],
|
||||
"last_is_overfull": last["is_overfull"],
|
||||
"last_date": last["date"],
|
||||
"days_since_last": (reference - last["date_value"]).days,
|
||||
"trend": _trend(numeric),
|
||||
"materials": materials,
|
||||
"recent_readings": recent,
|
||||
}
|
||||
|
||||
|
||||
def build(csv_path: Path) -> dict[str, Any]:
|
||||
rows = _read_rows(csv_path)
|
||||
reference = max(row["date_value"] for row in rows)
|
||||
date_from = min(row["date_value"] for row in rows)
|
||||
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in rows:
|
||||
grouped[row["location_id"]].append(row)
|
||||
containers = [
|
||||
_container(grouped[location_id], reference)
|
||||
for location_id in sorted(grouped)
|
||||
]
|
||||
containers.sort(key=container_sort_key)
|
||||
|
||||
material_counts = Counter(row["material_type"] for row in rows)
|
||||
material_overfull = Counter(
|
||||
row["material_type"] for row in rows if row["is_overfull"]
|
||||
)
|
||||
priority_counts = Counter(container["priority"] for container in containers)
|
||||
by_material = {
|
||||
name: {
|
||||
"readings": material_counts[name],
|
||||
"overfull": material_overfull[name],
|
||||
}
|
||||
for name in _material_names(set(material_counts))
|
||||
}
|
||||
|
||||
meta = {
|
||||
"source": csv_path.name,
|
||||
"generated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"rows": len(rows),
|
||||
"containers": len(containers),
|
||||
"material_series": len(
|
||||
{(row["location_id"], row["material_type"]) for row in rows}
|
||||
),
|
||||
"date_from": date_from.isoformat(),
|
||||
"date_to": reference.isoformat(),
|
||||
"reference_date": reference.isoformat(),
|
||||
"overfull_readings": sum(row["is_overfull"] for row in rows),
|
||||
"priority_rule": PRIORITY_RULE,
|
||||
"limitations": list(LIMITATIONS),
|
||||
}
|
||||
summary = {
|
||||
"priority_counts": {
|
||||
"high": priority_counts["high"],
|
||||
"medium": priority_counts["medium"],
|
||||
"low": priority_counts["low"],
|
||||
},
|
||||
"by_material": by_material,
|
||||
}
|
||||
return {"meta": meta, "summary": summary, "containers": containers}
|
||||
|
||||
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--csv",
|
||||
type=Path,
|
||||
default=config.prezero_csv(),
|
||||
help="raw fill-level CSV (git-ignored; default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out",
|
||||
type=Path,
|
||||
default=config.prezero_artifact(),
|
||||
help="artifact destination (default: %(default)s)",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
try:
|
||||
payload = build(args.csv)
|
||||
except BuildError as exc:
|
||||
print(f"build_prezero: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(
|
||||
json.dumps(payload, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
meta = payload["meta"]
|
||||
counts = payload["summary"]["priority_counts"]
|
||||
print(f"wrote {args.out} ({args.out.stat().st_size:,} bytes)")
|
||||
print(
|
||||
f"rows={meta['rows']} containers={meta['containers']} "
|
||||
f"material_series={meta['material_series']} "
|
||||
f"overfull={meta['overfull_readings']}"
|
||||
)
|
||||
print(
|
||||
f"date_range={meta['date_from']}..{meta['date_to']} "
|
||||
f"high={counts['high']} medium={counts['medium']} low={counts['low']}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user