#!/usr/bin/env python3 """Build ``artifacts/prezero_priority.json`` from the raw fill-level export. The raw CSV is challenge data and stays git-ignored. The derived artifact is committed and is the source of truth for the internal ``/prezero`` routes. Priority rule (contract, transparent, no invented labels): high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25) medium: overfull_count >= 1 and not high low: overfull_count == 0 There are no coordinates in the dataset: this is a service-priority proof, not a route. ``priority_score`` is a sorting/display convenience; the tier is the contract. Examples: uv run python scripts/build_prezero.py uv run python scripts/build_prezero.py --csv data/raw/glass_fill_levels_anonymized_f.csv """ from __future__ import annotations import argparse import csv import json import sys from collections import Counter, defaultdict from datetime import UTC, date, datetime from pathlib import Path from typing import Any import numpy as np from pulsecast import config from pulsecast.prezero import container_sort_key, priority_tier REQUIRED_COLUMNS = ("Location_id", "material_type", "date", "fill_level_perc") MATERIAL_ORDER = ("white", "green", "brown") PRIORITY_RULE = ( "high: overfull_count >= 5 or (overfull_count >= 2 and overfull_rate >= 0.25); " "medium: overfull_count >= 1 and not high; low: 0" ) LIMITATIONS = [ "No coordinates in the dataset; this is a priority proof, not a route.", "No container capacity or depot data.", "Overfull is a reported state, not a measured volume.", ] class BuildError(ValueError): """The input CSV does not satisfy the PreZero contract.""" def _normalized_columns(fieldnames: list[str] | None, path: Path) -> dict[str, str]: """Map BOM/whitespace-normalized names back to the CSV's raw names.""" if not fieldnames: raise BuildError(f"{path}: missing header row") mapping: dict[str, str] = {} for raw in fieldnames: name = str(raw).strip().lstrip("\ufeff") if not name: raise BuildError(f"{path}: blank column name") if name in mapping: raise BuildError(f"{path}: duplicate column after trimming: {name}") mapping[name] = raw missing = [name for name in REQUIRED_COLUMNS if name not in mapping] if missing: raise BuildError( f"{path}: missing required columns: {', '.join(missing)}; " f"found: {', '.join(mapping)}" ) return mapping def _read_rows(path: Path) -> list[dict[str, Any]]: """Read the export as strings, normalize headers/cells, validate every row.""" try: handle = path.open("r", encoding="utf-8-sig", newline="") except FileNotFoundError as exc: raise BuildError( f"CSV not found: {path}. Copy the raw export there or pass --csv." ) from exc except OSError as exc: raise BuildError(f"cannot read {path}: {exc}") from exc rows: list[dict[str, Any]] = [] with handle: reader = csv.DictReader(handle) columns = _normalized_columns(reader.fieldnames, path) for line_number, raw_row in enumerate(reader, start=2): row = { name: str(raw_row.get(source) or "").strip() for name, source in columns.items() } blank = [name for name in REQUIRED_COLUMNS if not row[name]] if blank: raise BuildError(f"{path}:{line_number}: blank values in {', '.join(blank)}") try: reading_date = date.fromisoformat(row["date"]) except ValueError as exc: raise BuildError( f"{path}:{line_number}: invalid date {row['date']!r}" ) from exc fill_text = row["fill_level_perc"] if fill_text == "overfull": fill: int | None = None is_overfull = True else: try: fill = int(fill_text) except ValueError as exc: raise BuildError( f"{path}:{line_number}: fill_level_perc must be an integer " f"0-100 or 'overfull', got {fill_text!r}" ) from exc if not 0 <= fill <= 100: raise BuildError( f"{path}:{line_number}: fill_level_perc out of range: {fill}" ) is_overfull = False rows.append( { "index": len(rows), "location_id": row["Location_id"], "material_type": row["material_type"], "date": row["date"], "date_value": reading_date, "fill_level_perc": fill_text, "fill": fill, "is_overfull": is_overfull, } ) if not rows: raise BuildError(f"{path}: no data rows") return rows def _ordered(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: """Chronological; input order breaks ties on the same date.""" return sorted(rows, key=lambda row: (row["date_value"], row["index"])) def _trend(numeric: list[int]) -> float | None: """Least-squares slope over the last up to 3 numeric readings (points/read).""" if len(numeric) < 2: return None tail = numeric[-3:] x_mean = (len(tail) - 1) / 2 y_mean = sum(tail) / len(tail) numerator = sum((x - x_mean) * (y - y_mean) for x, y in enumerate(tail)) denominator = sum((x - x_mean) ** 2 for x in range(len(tail))) slope = numerator / denominator return round(slope, 2) if slope else 0.0 def _material_names(materials: set[str]) -> list[str]: known = [name for name in MATERIAL_ORDER if name in materials] return known + sorted(materials - set(known)) def _material_summary(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: ordered = _ordered(rows) last = ordered[-1] return { "readings": len(ordered), "overfull_count": sum(row["is_overfull"] for row in ordered), "last_fill": last["fill"], "last_is_overfull": last["is_overfull"], "last_date": last["date"], } def _container(rows: list[dict[str, Any]], reference: date) -> dict[str, Any]: ordered = _ordered(rows) readings = len(ordered) overfull_count = sum(row["is_overfull"] for row in ordered) overfull_rate = overfull_count / readings numeric = [row["fill"] for row in ordered if row["fill"] is not None] last = ordered[-1] by_material: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in ordered: by_material[row["material_type"]].append(row) materials = { name: _material_summary(by_material[name]) for name in _material_names(set(by_material)) } p95_fill = round(float(np.percentile(numeric, 95)), 2) if numeric else None mean_fill = round(sum(numeric) / len(numeric), 2) if numeric else None recent = [ { "date": row["date"], "material_type": row["material_type"], "fill_level_perc": row["fill_level_perc"], "is_overfull": row["is_overfull"], } for row in reversed(ordered[-10:]) ] return { "location_id": ordered[0]["location_id"], "priority": priority_tier(overfull_count, overfull_rate), "priority_score": round(overfull_count + 2 * overfull_rate, 4), "readings": readings, "overfull_count": overfull_count, "overfull_rate": overfull_rate, "numeric_count": len(numeric), "p95_fill": p95_fill, "max_fill": max(numeric) if numeric else None, "mean_fill": mean_fill, "last_fill": last["fill"], "last_is_overfull": last["is_overfull"], "last_date": last["date"], "days_since_last": (reference - last["date_value"]).days, "trend": _trend(numeric), "materials": materials, "recent_readings": recent, } def build(csv_path: Path) -> dict[str, Any]: rows = _read_rows(csv_path) reference = max(row["date_value"] for row in rows) date_from = min(row["date_value"] for row in rows) grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) for row in rows: grouped[row["location_id"]].append(row) containers = [ _container(grouped[location_id], reference) for location_id in sorted(grouped) ] containers.sort(key=container_sort_key) material_counts = Counter(row["material_type"] for row in rows) material_overfull = Counter( row["material_type"] for row in rows if row["is_overfull"] ) priority_counts = Counter(container["priority"] for container in containers) by_material = { name: { "readings": material_counts[name], "overfull": material_overfull[name], } for name in _material_names(set(material_counts)) } meta = { "source": csv_path.name, "generated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "rows": len(rows), "containers": len(containers), "material_series": len( {(row["location_id"], row["material_type"]) for row in rows} ), "date_from": date_from.isoformat(), "date_to": reference.isoformat(), "reference_date": reference.isoformat(), "overfull_readings": sum(row["is_overfull"] for row in rows), "priority_rule": PRIORITY_RULE, "limitations": list(LIMITATIONS), } summary = { "priority_counts": { "high": priority_counts["high"], "medium": priority_counts["medium"], "low": priority_counts["low"], }, "by_material": by_material, } return {"meta": meta, "summary": summary, "containers": containers} def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "--csv", type=Path, default=config.prezero_csv(), help="raw fill-level CSV (git-ignored; default: %(default)s)", ) parser.add_argument( "--out", type=Path, default=config.prezero_artifact(), help="artifact destination (default: %(default)s)", ) return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) try: payload = build(args.csv) except BuildError as exc: print(f"build_prezero: {exc}", file=sys.stderr) return 1 args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text( json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", ) meta = payload["meta"] counts = payload["summary"]["priority_counts"] print(f"wrote {args.out} ({args.out.stat().st_size:,} bytes)") print( f"rows={meta['rows']} containers={meta['containers']} " f"material_series={meta['material_series']} " f"overfull={meta['overfull_readings']}" ) print( f"date_range={meta['date_from']}..{meta['date_to']} " f"high={counts['high']} medium={counts['medium']} low={counts['low']}" ) return 0 if __name__ == "__main__": raise SystemExit(main())