- make small trains from a CSV and prints metrics + confusion matrix - make figures renders dependency-free SVGs from metrics.json - scripts/e2e_train.py boots a real server and checks snapshot, robustness, auth, one-job-at-a-time, and both training jobs with hot-swap - CI/ruff/compileall now include scripts/
154 lines
5.5 KiB
Python
154 lines
5.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Train and evaluate the small "busy hour" classifier from a CSV.
|
|
|
|
This is the repeatable artifact behind ``make small`` and the runtime
|
|
``model=small`` training job. It prints every number reported in
|
|
``docs/small-model.md``; ``--save`` bakes the bundle and metrics into
|
|
``artifacts/`` exactly like ``make train`` does.
|
|
|
|
Examples:
|
|
uv run python scripts/train_small.py
|
|
uv run python scripts/train_small.py --csv new-data.csv --all
|
|
uv run python scripts/train_small.py --save --json
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import joblib
|
|
import pandas as pd
|
|
|
|
from pulsecast import config
|
|
from pulsecast.small import load_hourly_csv, train_small_from_csv
|
|
|
|
|
|
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument(
|
|
"csv_positional",
|
|
nargs="?",
|
|
metavar="CSV",
|
|
help=f"CSV to train from (default: {config.raw_csv_path()})",
|
|
)
|
|
parser.add_argument(
|
|
"--csv",
|
|
dest="csv_option",
|
|
help="CSV to train from; overrides the positional argument",
|
|
)
|
|
parser.add_argument(
|
|
"--start",
|
|
default=config.MULTI_SENSOR_START,
|
|
help=(
|
|
"drop rows before this UTC timestamp (default: %(default)s, the "
|
|
"8-sensor challenge window); ignored with --all"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--all",
|
|
action="store_true",
|
|
help="use every row in the CSV, including density-01's older history",
|
|
)
|
|
parser.add_argument(
|
|
"--save",
|
|
action="store_true",
|
|
help="write artifacts/small_model.joblib and artifacts/small_metrics.json",
|
|
)
|
|
parser.add_argument(
|
|
"--json",
|
|
action="store_true",
|
|
help="print the full metrics payload as JSON after the summary",
|
|
)
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def _confusion_lines(matrix: dict) -> list[str]:
|
|
tn, fp, fn, tp = matrix["tn"], matrix["fp"], matrix["fn"], matrix["tp"]
|
|
total = tn + fp + fn + tp
|
|
specificity = tn / (tn + fp) if tn + fp else 0.0
|
|
negative_predictive = tn / (tn + fn) if tn + fn else 0.0
|
|
return [
|
|
"Confusion matrix (rows = truth, columns = prediction)",
|
|
f" not busy: {tn:5d} {fp:5d}",
|
|
f" busy : {fn:5d} {tp:5d}",
|
|
f" n={total} specificity={specificity:.3f} NPV={negative_predictive:.3f}",
|
|
]
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = _parse_args(argv)
|
|
path = Path(args.csv_option or args.csv_positional or config.raw_csv_path())
|
|
if not path.exists():
|
|
print(f"CSV not found: {path}", file=sys.stderr)
|
|
return 2
|
|
|
|
start = None if args.all else pd.Timestamp(args.start)
|
|
started = time.perf_counter()
|
|
try:
|
|
bundle, payload = train_small_from_csv(path, start=start)
|
|
except (ValueError, FileNotFoundError) as exc:
|
|
print(f"training failed: {exc}", file=sys.stderr)
|
|
return 1
|
|
elapsed = time.perf_counter() - started
|
|
|
|
hourly = load_hourly_csv(path)
|
|
if start is not None:
|
|
hourly = hourly[hourly["t"] >= start]
|
|
metrics = payload["metrics"]
|
|
|
|
print(f"CSV {path}")
|
|
print(f"Rows {len(hourly):,} hourly rows, {metrics['sensors']} sensor(s)")
|
|
print(f"Window {payload['train_from']} .. {payload['test_to']}")
|
|
print(f"Split (time) train {payload['train_from']} .. {payload['train_to']}"
|
|
f" ({metrics['n_train']:,} rows)")
|
|
print(f" test {payload['test_from']} .. {payload['test_to']}"
|
|
f" ({metrics['n_test']:,} rows)")
|
|
print("Target busy = hourly density > sensor p75 (training window only)")
|
|
print(f"Features {', '.join(payload['features'])}")
|
|
print("Model standardized logistic regression (class_weight=balanced)")
|
|
print(f"Model version {payload['model_version']}")
|
|
print()
|
|
print("Metrics")
|
|
print(f" accuracy {metrics['accuracy']:.3f}")
|
|
print(f" precision {metrics['precision']:.3f}")
|
|
print(f" recall {metrics['recall']:.3f}")
|
|
print(f" f1 {metrics['f1']:.3f}")
|
|
print(f" roc_auc {metrics['roc_auc'] if metrics['roc_auc'] is not None else 'n/a'}")
|
|
print(f" base rate {metrics['positive_rate']:.3f}")
|
|
print()
|
|
print("\n".join(_confusion_lines(payload["confusion_matrix"])))
|
|
print()
|
|
print("Per sensor (holdout)")
|
|
print(f" {'sensor':<24} {'n':>6} {'pos':>6} {'acc':>6} {'prec':>6} {'rec':>6} {'f1':>6}")
|
|
for row in payload.get("per_sensor", []):
|
|
print(
|
|
f" {row['sensor']:<24} {row['n']:>6} {row['positive_rate']:>6.3f}"
|
|
f" {row['accuracy']:>6.3f} {row['precision']:>6.3f}"
|
|
f" {row['recall']:>6.3f} {row['f1']:>6.3f}"
|
|
)
|
|
print()
|
|
print(f"Fit time {elapsed:.2f} s (contract: < 5 s)")
|
|
|
|
if args.save:
|
|
artifacts = config.artifacts_dir()
|
|
artifacts.mkdir(parents=True, exist_ok=True)
|
|
joblib.dump(bundle, artifacts / "small_model.joblib")
|
|
(artifacts / "small_metrics.json").write_text(
|
|
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
|
)
|
|
print(f"Saved {artifacts / 'small_model.joblib'}")
|
|
print(f" {artifacts / 'small_metrics.json'}")
|
|
|
|
if args.json:
|
|
print()
|
|
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|