feat: repeatable small-model, figures, and training e2e scripts
- make small trains from a CSV and prints metrics + confusion matrix - make figures renders dependency-free SVGs from metrics.json - scripts/e2e_train.py boots a real server and checks snapshot, robustness, auth, one-job-at-a-time, and both training jobs with hot-swap - CI/ruff/compileall now include scripts/
This commit is contained in:
1 parent
000aaed7a5
commit
2bcf66c550
5 files changed
+952
-6
No files matched your search
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Train and evaluate the small "busy hour" classifier from a CSV.
|
||||
|
||||
This is the repeatable artifact behind ``make small`` and the runtime
|
||||
``model=small`` training job. It prints every number reported in
|
||||
``docs/small-model.md``; ``--save`` bakes the bundle and metrics into
|
||||
``artifacts/`` exactly like ``make train`` does.
|
||||
|
||||
Examples:
|
||||
uv run python scripts/train_small.py
|
||||
uv run python scripts/train_small.py --csv new-data.csv --all
|
||||
uv run python scripts/train_small.py --save --json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import joblib
|
||||
import pandas as pd
|
||||
|
||||
from pulsecast import config
|
||||
from pulsecast.small import load_hourly_csv, train_small_from_csv
|
||||
|
||||
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument(
|
||||
"csv_positional",
|
||||
nargs="?",
|
||||
metavar="CSV",
|
||||
help=f"CSV to train from (default: {config.raw_csv_path()})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--csv",
|
||||
dest="csv_option",
|
||||
help="CSV to train from; overrides the positional argument",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--start",
|
||||
default=config.MULTI_SENSOR_START,
|
||||
help=(
|
||||
"drop rows before this UTC timestamp (default: %(default)s, the "
|
||||
"8-sensor challenge window); ignored with --all"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="use every row in the CSV, including density-01's older history",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save",
|
||||
action="store_true",
|
||||
help="write artifacts/small_model.joblib and artifacts/small_metrics.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="print the full metrics payload as JSON after the summary",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def _confusion_lines(matrix: dict) -> list[str]:
|
||||
tn, fp, fn, tp = matrix["tn"], matrix["fp"], matrix["fn"], matrix["tp"]
|
||||
total = tn + fp + fn + tp
|
||||
specificity = tn / (tn + fp) if tn + fp else 0.0
|
||||
negative_predictive = tn / (tn + fn) if tn + fn else 0.0
|
||||
return [
|
||||
"Confusion matrix (rows = truth, columns = prediction)",
|
||||
f" not busy: {tn:5d} {fp:5d}",
|
||||
f" busy : {fn:5d} {tp:5d}",
|
||||
f" n={total} specificity={specificity:.3f} NPV={negative_predictive:.3f}",
|
||||
]
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
path = Path(args.csv_option or args.csv_positional or config.raw_csv_path())
|
||||
if not path.exists():
|
||||
print(f"CSV not found: {path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
start = None if args.all else pd.Timestamp(args.start)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
bundle, payload = train_small_from_csv(path, start=start)
|
||||
except (ValueError, FileNotFoundError) as exc:
|
||||
print(f"training failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
elapsed = time.perf_counter() - started
|
||||
|
||||
hourly = load_hourly_csv(path)
|
||||
if start is not None:
|
||||
hourly = hourly[hourly["t"] >= start]
|
||||
metrics = payload["metrics"]
|
||||
|
||||
print(f"CSV {path}")
|
||||
print(f"Rows {len(hourly):,} hourly rows, {metrics['sensors']} sensor(s)")
|
||||
print(f"Window {payload['train_from']} .. {payload['test_to']}")
|
||||
print(f"Split (time) train {payload['train_from']} .. {payload['train_to']}"
|
||||
f" ({metrics['n_train']:,} rows)")
|
||||
print(f" test {payload['test_from']} .. {payload['test_to']}"
|
||||
f" ({metrics['n_test']:,} rows)")
|
||||
print("Target busy = hourly density > sensor p75 (training window only)")
|
||||
print(f"Features {', '.join(payload['features'])}")
|
||||
print("Model standardized logistic regression (class_weight=balanced)")
|
||||
print(f"Model version {payload['model_version']}")
|
||||
print()
|
||||
print("Metrics")
|
||||
print(f" accuracy {metrics['accuracy']:.3f}")
|
||||
print(f" precision {metrics['precision']:.3f}")
|
||||
print(f" recall {metrics['recall']:.3f}")
|
||||
print(f" f1 {metrics['f1']:.3f}")
|
||||
print(f" roc_auc {metrics['roc_auc'] if metrics['roc_auc'] is not None else 'n/a'}")
|
||||
print(f" base rate {metrics['positive_rate']:.3f}")
|
||||
print()
|
||||
print("\n".join(_confusion_lines(payload["confusion_matrix"])))
|
||||
print()
|
||||
print("Per sensor (holdout)")
|
||||
print(f" {'sensor':<24} {'n':>6} {'pos':>6} {'acc':>6} {'prec':>6} {'rec':>6} {'f1':>6}")
|
||||
for row in payload.get("per_sensor", []):
|
||||
print(
|
||||
f" {row['sensor']:<24} {row['n']:>6} {row['positive_rate']:>6.3f}"
|
||||
f" {row['accuracy']:>6.3f} {row['precision']:>6.3f}"
|
||||
f" {row['recall']:>6.3f} {row['f1']:>6.3f}"
|
||||
)
|
||||
print()
|
||||
print(f"Fit time {elapsed:.2f} s (contract: < 5 s)")
|
||||
|
||||
if args.save:
|
||||
artifacts = config.artifacts_dir()
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
joblib.dump(bundle, artifacts / "small_model.joblib")
|
||||
(artifacts / "small_metrics.json").write_text(
|
||||
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
print(f"Saved {artifacts / 'small_model.joblib'}")
|
||||
print(f" {artifacts / 'small_metrics.json'}")
|
||||
|
||||
if args.json:
|
||||
print()
|
||||
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user