The small-model page said make small reproduced every number, but the threshold table and several derived values only appear in the JSON payload. Say what the commands actually print, fix the stale weather-cache docstring, and lead the e2e evidence page with what it checks.
154 lines
5.5 KiB
Python
154 lines
5.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Train and evaluate the small "busy hour" classifier from a CSV.
|
|
|
|
This is the repeatable artifact behind ``make small`` and the runtime
|
|
``model=small`` training job. It prints the headline metrics and per-sensor
|
|
table used in ``docs/small-model.md``; ``--save`` bakes the bundle and metrics
|
|
into ``artifacts/`` exactly like ``make train`` does.
|
|
|
|
Examples:
|
|
uv run python scripts/train_small.py
|
|
uv run python scripts/train_small.py --csv new-data.csv --all
|
|
uv run python scripts/train_small.py --save --json
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import joblib
|
|
import pandas as pd
|
|
|
|
from pulsecast import config
|
|
from pulsecast.small import load_hourly_csv, train_small_from_csv
|
|
|
|
|
|
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument(
|
|
"csv_positional",
|
|
nargs="?",
|
|
metavar="CSV",
|
|
help=f"CSV to train from (default: {config.raw_csv_path()})",
|
|
)
|
|
parser.add_argument(
|
|
"--csv",
|
|
dest="csv_option",
|
|
help="CSV to train from; overrides the positional argument",
|
|
)
|
|
parser.add_argument(
|
|
"--start",
|
|
default=config.MULTI_SENSOR_START,
|
|
help=(
|
|
"drop rows before this UTC timestamp (default: %(default)s, the "
|
|
"8-sensor challenge window); ignored with --all"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--all",
|
|
action="store_true",
|
|
help="use every row in the CSV, including density-01's older history",
|
|
)
|
|
parser.add_argument(
|
|
"--save",
|
|
action="store_true",
|
|
help="write artifacts/small_model.joblib and artifacts/small_metrics.json",
|
|
)
|
|
parser.add_argument(
|
|
"--json",
|
|
action="store_true",
|
|
help="print the full metrics payload as JSON after the summary",
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def _confusion_lines(matrix: dict) -> list[str]:
|
|
tn, fp, fn, tp = matrix["tn"], matrix["fp"], matrix["fn"], matrix["tp"]
|
|
total = tn + fp + fn + tp
|
|
specificity = tn / (tn + fp) if tn + fp else 0.0
|
|
negative_predictive = tn / (tn + fn) if tn + fn else 0.0
|
|
return [
|
|
"Confusion matrix (rows = truth, columns = prediction)",
|
|
f" not busy: {tn:5d} {fp:5d}",
|
|
f" busy : {fn:5d} {tp:5d}",
|
|
f" n={total} specificity={specificity:.3f} NPV={negative_predictive:.3f}",
|
|
]
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = _parse_args(argv)
|
|
path = Path(args.csv_option or args.csv_positional or config.raw_csv_path())
|
|
if not path.exists():
|
|
print(f"CSV not found: {path}", file=sys.stderr)
|
|
return 2
|
|
|
|
start = None if args.all else pd.Timestamp(args.start)
|
|
started = time.perf_counter()
|
|
try:
|
|
bundle, payload = train_small_from_csv(path, start=start)
|
|
except (ValueError, FileNotFoundError) as exc:
|
|
print(f"training failed: {exc}", file=sys.stderr)
|
|
return 1
|
|
elapsed = time.perf_counter() - started
|
|
|
|
hourly = load_hourly_csv(path)
|
|
if start is not None:
|
|
hourly = hourly[hourly["t"] >= start]
|
|
metrics = payload["metrics"]
|
|
|
|
print(f"CSV {path}")
|
|
print(f"Rows {len(hourly):,} hourly rows, {metrics['sensors']} sensor(s)")
|
|
print(f"Window {payload['train_from']} .. {payload['test_to']}")
|
|
print(f"Split (time) train {payload['train_from']} .. {payload['train_to']}"
|
|
f" ({metrics['n_train']:,} rows)")
|
|
print(f" test {payload['test_from']} .. {payload['test_to']}"
|
|
f" ({metrics['n_test']:,} rows)")
|
|
print("Target busy = hourly density > sensor p75 (training window only)")
|
|
print(f"Features {', '.join(payload['features'])}")
|
|
print("Model standardized logistic regression (class_weight=balanced)")
|
|
print(f"Model version {payload['model_version']}")
|
|
print()
|
|
print("Metrics")
|
|
print(f" accuracy {metrics['accuracy']:.3f}")
|
|
print(f" precision {metrics['precision']:.3f}")
|
|
print(f" recall {metrics['recall']:.3f}")
|
|
print(f" f1 {metrics['f1']:.3f}")
|
|
print(f" roc_auc {metrics['roc_auc'] if metrics['roc_auc'] is not None else 'n/a'}")
|
|
print(f" base rate {metrics['positive_rate']:.3f}")
|
|
print()
|
|
print("\n".join(_confusion_lines(payload["confusion_matrix"])))
|
|
print()
|
|
print("Per sensor (holdout)")
|
|
print(f" {'sensor':<24} {'n':>6} {'pos':>6} {'acc':>6} {'prec':>6} {'rec':>6} {'f1':>6}")
|
|
for row in payload.get("per_sensor", []):
|
|
print(
|
|
f" {row['sensor']:<24} {row['n']:>6} {row['positive_rate']:>6.3f}"
|
|
f" {row['accuracy']:>6.3f} {row['precision']:>6.3f}"
|
|
f" {row['recall']:>6.3f} {row['f1']:>6.3f}"
|
|
)
|
|
print()
|
|
print(f"Fit time {elapsed:.2f} s (contract: < 5 s)")
|
|
|
|
if args.save:
|
|
artifacts = config.artifacts_dir()
|
|
artifacts.mkdir(parents=True, exist_ok=True)
|
|
joblib.dump(bundle, artifacts / "small_model.joblib")
|
|
(artifacts / "small_metrics.json").write_text(
|
|
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
|
)
|
|
print(f"Saved {artifacts / 'small_model.joblib'}")
|
|
print(f" {artifacts / 'small_metrics.json'}")
|
|
|
|
if args.json:
|
|
print()
|
|
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|