Build artifacts/prezero_priority.json from the anonymized glass fill-level export (11,701 rows, 449 containers, 1,338 location/material series). The contract tier rule yields high 88, medium 248, low 113 with 1,192 overfull readings.
Add /prezero/summary, /prezero/priority, and /prezero/containers/{id} to the internal FastAPI service, with startup validation, filtering/pagination, recent reading history, and 503 isolation when the artifact is missing. Raw CSV stays gitignored; the derived 1.1 MB artifact is the source of truth. This is a priority proof, not a route: the dataset has no coordinates.
330 lines
11 KiB
Python
330 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Build ``artifacts/prezero_priority.json`` from the raw fill-level export.
|
|
|
|
The raw CSV is challenge data and stays git-ignored. The derived artifact is
|
|
committed and is the source of truth for the internal ``/prezero`` routes.
|
|
|
|
Priority rule (contract, transparent, no invented labels):
|
|
|
|
high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25)
|
|
medium: overfull_count >= 1 and not high
|
|
low: overfull_count == 0
|
|
|
|
There are no coordinates in the dataset: this is a service-priority proof, not
|
|
a route. ``priority_score`` is a sorting/display convenience; the tier is the
|
|
contract.
|
|
|
|
Examples:
|
|
uv run python scripts/build_prezero.py
|
|
uv run python scripts/build_prezero.py --csv data/raw/glass_fill_levels_anonymized_f.csv
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import sys
|
|
from collections import Counter, defaultdict
|
|
from datetime import UTC, date, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
from pulsecast import config
|
|
from pulsecast.prezero import container_sort_key, priority_tier
|
|
|
|
REQUIRED_COLUMNS = ("Location_id", "material_type", "date", "fill_level_perc")
|
|
MATERIAL_ORDER = ("white", "green", "brown")
|
|
PRIORITY_RULE = (
|
|
"high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25); "
|
|
"medium: overfull_count >= 1 and not high; low: 0"
|
|
)
|
|
LIMITATIONS = [
|
|
"No coordinates in the dataset; this is a priority proof, not a route.",
|
|
"No container capacity or depot data.",
|
|
"Overfull is a reported state, not a measured volume.",
|
|
]
|
|
|
|
|
|
class BuildError(ValueError):
|
|
"""The input CSV does not satisfy the PreZero contract."""
|
|
|
|
|
|
def _normalized_columns(fieldnames: list[str] | None, path: Path) -> dict[str, str]:
|
|
"""Map BOM/whitespace-normalized names back to the CSV's raw names."""
|
|
if not fieldnames:
|
|
raise BuildError(f"{path}: missing header row")
|
|
mapping: dict[str, str] = {}
|
|
for raw in fieldnames:
|
|
name = str(raw).strip().lstrip("\ufeff")
|
|
if not name:
|
|
raise BuildError(f"{path}: blank column name")
|
|
if name in mapping:
|
|
raise BuildError(f"{path}: duplicate column after trimming: {name}")
|
|
mapping[name] = raw
|
|
missing = [name for name in REQUIRED_COLUMNS if name not in mapping]
|
|
if missing:
|
|
raise BuildError(
|
|
f"{path}: missing required columns: {', '.join(missing)}; "
|
|
f"found: {', '.join(mapping)}"
|
|
)
|
|
return mapping
|
|
|
|
|
|
def _read_rows(path: Path) -> list[dict[str, Any]]:
|
|
"""Read the export as strings, normalize headers/cells, validate every row."""
|
|
try:
|
|
handle = path.open("r", encoding="utf-8-sig", newline="")
|
|
except FileNotFoundError as exc:
|
|
raise BuildError(
|
|
f"CSV not found: {path}. Copy the raw export there or pass --csv."
|
|
) from exc
|
|
except OSError as exc:
|
|
raise BuildError(f"cannot read {path}: {exc}") from exc
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
with handle:
|
|
reader = csv.DictReader(handle)
|
|
columns = _normalized_columns(reader.fieldnames, path)
|
|
for line_number, raw_row in enumerate(reader, start=2):
|
|
row = {
|
|
name: str(raw_row.get(source) or "").strip()
|
|
for name, source in columns.items()
|
|
}
|
|
blank = [name for name in REQUIRED_COLUMNS if not row[name]]
|
|
if blank:
|
|
raise BuildError(f"{path}:{line_number}: blank values in {', '.join(blank)}")
|
|
|
|
try:
|
|
reading_date = date.fromisoformat(row["date"])
|
|
except ValueError as exc:
|
|
raise BuildError(
|
|
f"{path}:{line_number}: invalid date {row['date']!r}"
|
|
) from exc
|
|
|
|
fill_text = row["fill_level_perc"]
|
|
if fill_text == "overfull":
|
|
fill: int | None = None
|
|
is_overfull = True
|
|
else:
|
|
try:
|
|
fill = int(fill_text)
|
|
except ValueError as exc:
|
|
raise BuildError(
|
|
f"{path}:{line_number}: fill_level_perc must be an integer "
|
|
f"0-100 or 'overfull', got {fill_text!r}"
|
|
) from exc
|
|
if not 0 <= fill <= 100:
|
|
raise BuildError(
|
|
f"{path}:{line_number}: fill_level_perc out of range: {fill}"
|
|
)
|
|
is_overfull = False
|
|
|
|
rows.append(
|
|
{
|
|
"index": len(rows),
|
|
"location_id": row["Location_id"],
|
|
"material_type": row["material_type"],
|
|
"date": row["date"],
|
|
"date_value": reading_date,
|
|
"fill_level_perc": fill_text,
|
|
"fill": fill,
|
|
"is_overfull": is_overfull,
|
|
}
|
|
)
|
|
if not rows:
|
|
raise BuildError(f"{path}: no data rows")
|
|
return rows
|
|
|
|
|
|
def _ordered(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Chronological; input order breaks ties on the same date."""
|
|
return sorted(rows, key=lambda row: (row["date_value"], row["index"]))
|
|
|
|
|
|
def _trend(numeric: list[int]) -> float | None:
|
|
"""Least-squares slope over the last up to 3 numeric readings (points/read)."""
|
|
if len(numeric) < 2:
|
|
return None
|
|
tail = numeric[-3:]
|
|
x_mean = (len(tail) - 1) / 2
|
|
y_mean = sum(tail) / len(tail)
|
|
numerator = sum((x - x_mean) * (y - y_mean) for x, y in enumerate(tail))
|
|
denominator = sum((x - x_mean) ** 2 for x in range(len(tail)))
|
|
slope = numerator / denominator
|
|
return round(slope, 2) if slope else 0.0
|
|
|
|
|
|
def _material_names(materials: set[str]) -> list[str]:
|
|
known = [name for name in MATERIAL_ORDER if name in materials]
|
|
return known + sorted(materials - set(known))
|
|
|
|
|
|
def _material_summary(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
ordered = _ordered(rows)
|
|
last = ordered[-1]
|
|
return {
|
|
"readings": len(ordered),
|
|
"overfull_count": sum(row["is_overfull"] for row in ordered),
|
|
"last_fill": last["fill"],
|
|
"last_is_overfull": last["is_overfull"],
|
|
"last_date": last["date"],
|
|
}
|
|
|
|
|
|
def _container(rows: list[dict[str, Any]], reference: date) -> dict[str, Any]:
|
|
ordered = _ordered(rows)
|
|
readings = len(ordered)
|
|
overfull_count = sum(row["is_overfull"] for row in ordered)
|
|
overfull_rate = overfull_count / readings
|
|
numeric = [row["fill"] for row in ordered if row["fill"] is not None]
|
|
last = ordered[-1]
|
|
|
|
by_material: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for row in ordered:
|
|
by_material[row["material_type"]].append(row)
|
|
materials = {
|
|
name: _material_summary(by_material[name])
|
|
for name in _material_names(set(by_material))
|
|
}
|
|
|
|
p95_fill = round(float(np.percentile(numeric, 95)), 2) if numeric else None
|
|
mean_fill = round(sum(numeric) / len(numeric), 2) if numeric else None
|
|
recent = [
|
|
{
|
|
"date": row["date"],
|
|
"material_type": row["material_type"],
|
|
"fill_level_perc": row["fill_level_perc"],
|
|
"is_overfull": row["is_overfull"],
|
|
}
|
|
for row in reversed(ordered[-10:])
|
|
]
|
|
|
|
return {
|
|
"location_id": ordered[0]["location_id"],
|
|
"priority": priority_tier(overfull_count, overfull_rate),
|
|
"priority_score": round(overfull_count + 2 * overfull_rate, 4),
|
|
"readings": readings,
|
|
"overfull_count": overfull_count,
|
|
"overfull_rate": overfull_rate,
|
|
"numeric_count": len(numeric),
|
|
"p95_fill": p95_fill,
|
|
"max_fill": max(numeric) if numeric else None,
|
|
"mean_fill": mean_fill,
|
|
"last_fill": last["fill"],
|
|
"last_is_overfull": last["is_overfull"],
|
|
"last_date": last["date"],
|
|
"days_since_last": (reference - last["date_value"]).days,
|
|
"trend": _trend(numeric),
|
|
"materials": materials,
|
|
"recent_readings": recent,
|
|
}
|
|
|
|
|
|
def build(csv_path: Path) -> dict[str, Any]:
|
|
rows = _read_rows(csv_path)
|
|
reference = max(row["date_value"] for row in rows)
|
|
date_from = min(row["date_value"] for row in rows)
|
|
|
|
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for row in rows:
|
|
grouped[row["location_id"]].append(row)
|
|
containers = [
|
|
_container(grouped[location_id], reference)
|
|
for location_id in sorted(grouped)
|
|
]
|
|
containers.sort(key=container_sort_key)
|
|
|
|
material_counts = Counter(row["material_type"] for row in rows)
|
|
material_overfull = Counter(
|
|
row["material_type"] for row in rows if row["is_overfull"]
|
|
)
|
|
priority_counts = Counter(container["priority"] for container in containers)
|
|
by_material = {
|
|
name: {
|
|
"readings": material_counts[name],
|
|
"overfull": material_overfull[name],
|
|
}
|
|
for name in _material_names(set(material_counts))
|
|
}
|
|
|
|
meta = {
|
|
"source": csv_path.name,
|
|
"generated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"rows": len(rows),
|
|
"containers": len(containers),
|
|
"material_series": len(
|
|
{(row["location_id"], row["material_type"]) for row in rows}
|
|
),
|
|
"date_from": date_from.isoformat(),
|
|
"date_to": reference.isoformat(),
|
|
"reference_date": reference.isoformat(),
|
|
"overfull_readings": sum(row["is_overfull"] for row in rows),
|
|
"priority_rule": PRIORITY_RULE,
|
|
"limitations": list(LIMITATIONS),
|
|
}
|
|
summary = {
|
|
"priority_counts": {
|
|
"high": priority_counts["high"],
|
|
"medium": priority_counts["medium"],
|
|
"low": priority_counts["low"],
|
|
},
|
|
"by_material": by_material,
|
|
}
|
|
return {"meta": meta, "summary": summary, "containers": containers}
|
|
|
|
|
|
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
)
|
|
parser.add_argument(
|
|
"--csv",
|
|
type=Path,
|
|
default=config.prezero_csv(),
|
|
help="raw fill-level CSV (git-ignored; default: %(default)s)",
|
|
)
|
|
parser.add_argument(
|
|
"--out",
|
|
type=Path,
|
|
default=config.prezero_artifact(),
|
|
help="artifact destination (default: %(default)s)",
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = _parse_args(argv)
|
|
try:
|
|
payload = build(args.csv)
|
|
except BuildError as exc:
|
|
print(f"build_prezero: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
|
args.out.write_text(
|
|
json.dumps(payload, indent=2, ensure_ascii=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
meta = payload["meta"]
|
|
counts = payload["summary"]["priority_counts"]
|
|
print(f"wrote {args.out} ({args.out.stat().st_size:,} bytes)")
|
|
print(
|
|
f"rows={meta['rows']} containers={meta['containers']} "
|
|
f"material_series={meta['material_series']} "
|
|
f"overfull={meta['overfull_readings']}"
|
|
)
|
|
print(
|
|
f"date_range={meta['date_from']}..{meta['date_to']} "
|
|
f"high={counts['high']} medium={counts['medium']} low={counts['low']}"
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|