feat: repeatable small-model, figures, and training e2e scripts
- make small trains from a CSV and prints metrics + confusion matrix - make figures renders dependency-free SVGs from metrics.json - scripts/e2e_train.py boots a real server and checks snapshot, robustness, auth, one-job-at-a-time, and both training jobs with hot-swap - CI/ruff/compileall now include scripts/
This commit is contained in:
1 parent
000aaed7a5
commit
2bcf66c550
5 files changed
+952
-6
No files matched your search
@@ -0,0 +1,523 @@
|
||||
#!/usr/bin/env python3
|
||||
"""End-to-end acceptance run against a real FastAPI process on port 18000.
|
||||
|
||||
Starts uvicorn with ``CONTROL_TOKEN``, then exercises robustness and the full
|
||||
training lifecycle:
|
||||
|
||||
- /healthz, /readyz, /meta, /bootstrap (8 sensors, hourly series, context)
|
||||
- /predict for every sensor x horizon x scenario (128 calls)
|
||||
- /simulate for every scenario with all sensors
|
||||
- auth, unknown job, unknown model, malformed CSV, one-job-at-a-time
|
||||
- POST /train model=small on a real CSV subset; poll to success; hot-swap
|
||||
- POST /train model=forecast on the full CSV; poll to success; hot-swap
|
||||
- /predict still works after each swap
|
||||
|
||||
Exit code is 0 only when every check passes. Figures printed to stdout are the
|
||||
measured numbers; ``--report`` writes the same result as JSON.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pandas as pd
|
||||
|
||||
from pulsecast import config
|
||||
|
||||
HORIZON_SIZES = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
|
||||
SCENARIOS = tuple(config.SCENARIOS)
|
||||
SENSORS = tuple(sorted(config.USABLE_SENSORS))
|
||||
|
||||
|
||||
class CheckFailed(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class Runner:
|
||||
def __init__(self) -> None:
|
||||
self.checks = 0
|
||||
self.results: dict = {}
|
||||
|
||||
def check(self, condition: bool, message: str) -> None:
|
||||
self.checks += 1
|
||||
if not condition:
|
||||
raise CheckFailed(message)
|
||||
print(f" [ok] {message}")
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _parse_ts(value: str) -> pd.Timestamp:
|
||||
return pd.Timestamp(value)
|
||||
|
||||
|
||||
def _seconds(start: str | None, end: str | None) -> float | None:
|
||||
if not start or not end:
|
||||
return None
|
||||
return round((_parse_ts(end) - _parse_ts(start)).total_seconds(), 3)
|
||||
|
||||
|
||||
def _wait_http(client: httpx.Client, url: str, timeout: float = 30.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
last: Exception | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response = client.get(url, timeout=2.0)
|
||||
if response.status_code == 200:
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 - startup race
|
||||
last = exc
|
||||
time.sleep(0.1)
|
||||
raise CheckFailed(f"server did not answer {url} within {timeout}s: {last}")
|
||||
|
||||
|
||||
def _wait_job(
|
||||
client: httpx.Client,
|
||||
job_id: str,
|
||||
headers: dict[str, str],
|
||||
*,
|
||||
timeout: float = 180.0,
|
||||
state: str | None = None,
|
||||
) -> dict:
|
||||
deadline = time.monotonic() + timeout
|
||||
last_status = None
|
||||
while time.monotonic() < deadline:
|
||||
response = client.get(f"/train/{job_id}", headers=headers)
|
||||
if response.status_code != 200:
|
||||
raise CheckFailed(f"GET /train/{job_id} -> {response.status_code}: {response.text}")
|
||||
job = response.json()
|
||||
last_status = job["status"]
|
||||
if state is not None and last_status == state:
|
||||
return job
|
||||
if state is None and last_status in {"succeeded", "failed"}:
|
||||
return job
|
||||
time.sleep(0.05)
|
||||
raise CheckFailed(f"job {job_id} did not reach {state or 'terminal'} (last={last_status})")
|
||||
|
||||
|
||||
def _job_headers(token: str) -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
def _submit(
|
||||
client: httpx.Client, headers: dict[str, str], path: Path, model: str
|
||||
) -> httpx.Response:
|
||||
return client.post(
|
||||
"/train",
|
||||
headers=headers,
|
||||
files={"file": (path.name, path.read_bytes(), "text/csv")},
|
||||
data={"model": model},
|
||||
timeout=120.0,
|
||||
)
|
||||
|
||||
|
||||
def _assert_points(points: list[dict], expected: int, label: str) -> None:
|
||||
if len(points) != expected:
|
||||
raise CheckFailed(f"{label}: expected {expected} points, got {len(points)}")
|
||||
previous = None
|
||||
for point in points:
|
||||
if set(point) != {"t", "v", "lo", "hi"}:
|
||||
raise CheckFailed(f"{label}: unexpected point keys {sorted(point)}")
|
||||
if not point["t"].endswith("Z"):
|
||||
raise CheckFailed(f"{label}: timestamp is not UTC RFC 3339: {point['t']}")
|
||||
if point["v"] < 0 or point["lo"] > point["v"] or point["v"] > point["hi"]:
|
||||
raise CheckFailed(f"{label}: invalid band {point}")
|
||||
stamp = _parse_ts(point["t"])
|
||||
if previous is not None and stamp <= previous:
|
||||
raise CheckFailed(f"{label}: timestamps are not increasing at {point['t']}")
|
||||
previous = stamp
|
||||
|
||||
|
||||
def _versions(payload: dict) -> dict:
|
||||
return {
|
||||
key: payload.get(key)
|
||||
for key in ("data_version", "model_version", "context_version")
|
||||
}
|
||||
|
||||
|
||||
def _build_subset(raw: Path, destination: Path, days: int = 60) -> tuple[Path, dict]:
|
||||
frame = pd.read_csv(raw)
|
||||
frame = frame[frame["type"].astype(str) == "densityData"].copy()
|
||||
frame["time"] = pd.to_datetime(frame["time"], utc=True)
|
||||
end = frame["time"].max()
|
||||
cut = end - pd.Timedelta(days=days)
|
||||
frame = frame[frame["time"] >= cut][["time", "name", "value"]]
|
||||
frame["time"] = frame["time"].dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
frame.to_csv(destination, index=False)
|
||||
return destination, {
|
||||
"rows": len(frame),
|
||||
"sensors": int(frame["name"].nunique()),
|
||||
"from": frame["time"].min(),
|
||||
"to": frame["time"].max(),
|
||||
"days": days,
|
||||
}
|
||||
|
||||
|
||||
def _port_free(port: int) -> bool:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(0.5)
|
||||
return sock.connect_ex(("127.0.0.1", port)) != 0
|
||||
|
||||
|
||||
def run(args: argparse.Namespace, runner: Runner) -> dict:
|
||||
raw = Path(args.csv)
|
||||
if not raw.exists():
|
||||
raise CheckFailed(f"CSV not found: {raw}")
|
||||
if not _port_free(args.port):
|
||||
raise CheckFailed(f"port {args.port} is already in use")
|
||||
|
||||
workdir = Path(tempfile.mkdtemp(prefix="pulsecast-e2e-"))
|
||||
models_dir = workdir / "models"
|
||||
uploads_dir = workdir / "uploads"
|
||||
subset_path, subset_info = _build_subset(raw, workdir / "small-subset.csv", args.days)
|
||||
garbage_path = workdir / "garbage.csv"
|
||||
garbage_path.write_text("not,a,pulsecast,file\n1,2,3,4\n")
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
CONTROL_TOKEN=args.token,
|
||||
PULSECAST_MODELS=str(models_dir),
|
||||
PULSECAST_UPLOADS=str(uploads_dir),
|
||||
)
|
||||
log_path = workdir / "server.log"
|
||||
log_handle = log_path.open("wb")
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
"pulsecast.server:app",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(args.port),
|
||||
"--log-level",
|
||||
"warning",
|
||||
]
|
||||
process = subprocess.Popen(command, env=env, stdout=log_handle, stderr=subprocess.STDOUT)
|
||||
report: dict = {
|
||||
"started_at": _utc_now(),
|
||||
"base_url": f"http://127.0.0.1:{args.port}",
|
||||
"subset": subset_info,
|
||||
"raw_csv_bytes": raw.stat().st_size,
|
||||
}
|
||||
|
||||
try:
|
||||
with httpx.Client(base_url=f"http://127.0.0.1:{args.port}", timeout=120.0) as client:
|
||||
_wait_http(client, "/healthz")
|
||||
headers = _job_headers(args.token)
|
||||
|
||||
print("Boot and snapshot")
|
||||
health = client.get("/healthz").json()
|
||||
runner.check(health == {"ok": True}, "/healthz returns {ok:true}")
|
||||
ready = client.get("/readyz")
|
||||
runner.check(ready.status_code == 200 and ready.json()["model"] is True, "/readyz 200 model=true")
|
||||
meta_before = client.get("/meta").json()
|
||||
runner.check(meta_before["coverage"]["sensors"] == 8, "/meta covers 8 sensors")
|
||||
runner.check(
|
||||
meta_before["metrics"]["mae"] < meta_before["metrics"]["baseline_mae"],
|
||||
"baked model beats seasonal naive on the holdout",
|
||||
)
|
||||
bootstrap = client.get("/bootstrap").json()
|
||||
runner.check(len(bootstrap["sensors"]) == 8, "/bootstrap returns 8 sensors")
|
||||
runner.check(
|
||||
set(bootstrap["series"]) == set(SENSORS),
|
||||
"/bootstrap series covers all 8 sensors",
|
||||
)
|
||||
series_points = sum(len(points) for points in bootstrap["series"].values())
|
||||
runner.check(series_points > 20_000, f"/bootstrap carries {series_points} hourly points")
|
||||
context = bootstrap["context"]
|
||||
runner.check(
|
||||
len(context["weather"]["forecast"]) == 168,
|
||||
"/bootstrap context has 168 forecast weather hours",
|
||||
)
|
||||
runner.check(bool(context["holidays"]), "/bootstrap context has holidays")
|
||||
runner.check(
|
||||
_versions(bootstrap) == _versions(meta_before),
|
||||
"/bootstrap versions match /meta",
|
||||
)
|
||||
report["versions_before"] = _versions(meta_before)
|
||||
report["bootstrap"] = {"sensors": 8, "series_points": series_points}
|
||||
|
||||
print("Robustness: /predict every sensor x horizon x scenario")
|
||||
predict_calls = 0
|
||||
for sensor in SENSORS:
|
||||
for horizon, count in HORIZON_SIZES.items():
|
||||
for scenario in SCENARIOS:
|
||||
response = client.post(
|
||||
"/predict",
|
||||
json={"sensor": sensor, "horizon": horizon, "scenario": scenario},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise CheckFailed(
|
||||
f"/predict {sensor}/{horizon}/{scenario} -> "
|
||||
f"{response.status_code}: {response.text}"
|
||||
)
|
||||
payload = response.json()
|
||||
_assert_points(
|
||||
payload["points"],
|
||||
count,
|
||||
f"/predict {sensor}/{horizon}/{scenario}",
|
||||
)
|
||||
predict_calls += 1
|
||||
runner.check(predict_calls == 128, "128 /predict combinations returned valid points")
|
||||
|
||||
print("Robustness: /simulate every scenario, all sensors")
|
||||
simulate_calls = 0
|
||||
for scenario in SCENARIOS:
|
||||
response = client.post(
|
||||
"/simulate", json={"scenario": scenario, "sensors": "all"}
|
||||
)
|
||||
if response.status_code != 200:
|
||||
raise CheckFailed(f"/simulate {scenario} -> {response.status_code}: {response.text}")
|
||||
payload = response.json()
|
||||
if set(payload["forecasts"]) != set(SENSORS):
|
||||
raise CheckFailed(f"/simulate {scenario}: sensor set mismatch")
|
||||
for sensor, by_horizon in payload["forecasts"].items():
|
||||
if set(by_horizon) != set(HORIZON_SIZES):
|
||||
raise CheckFailed(f"/simulate {scenario}/{sensor}: horizon set mismatch")
|
||||
for horizon, count in HORIZON_SIZES.items():
|
||||
_assert_points(
|
||||
by_horizon[horizon],
|
||||
count,
|
||||
f"/simulate {scenario}/{sensor}/{horizon}",
|
||||
)
|
||||
simulate_calls += 1
|
||||
runner.check(simulate_calls == 4, "4 /simulate scenarios x 8 sensors x 4 horizons")
|
||||
|
||||
cold = client.post(
|
||||
"/predict",
|
||||
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": -8.0, "precip_mm": 6.0}},
|
||||
).json()["points"]
|
||||
warm = client.post(
|
||||
"/predict",
|
||||
json={"sensor": SENSORS[0], "horizon": "48h", "weather": {"temp_c": 30.0, "precip_mm": 0.0}},
|
||||
).json()["points"]
|
||||
runner.check(cold != warm, "weather override changes the forecast")
|
||||
|
||||
print("Error handling")
|
||||
no_token = client.post("/predict", json={"sensor": "nope"})
|
||||
runner.check(no_token.status_code == 404, "/predict unknown sensor -> 404")
|
||||
unauthorized = _submit(client, {}, subset_path, "small")
|
||||
runner.check(
|
||||
unauthorized.status_code == 401
|
||||
and unauthorized.json()["error"]["code"] == "unauthorized",
|
||||
"/train without token -> 401 unauthorized",
|
||||
)
|
||||
bad_token = _submit(client, _job_headers("wrong-token"), subset_path, "small")
|
||||
runner.check(bad_token.status_code == 401, "/train with wrong token -> 401")
|
||||
unknown_job = client.get("/train/nope", headers=headers)
|
||||
runner.check(
|
||||
unknown_job.status_code == 404
|
||||
and unknown_job.json()["error"]["code"] == "not_found",
|
||||
"/train/nope -> 404 not_found",
|
||||
)
|
||||
bad_model = _submit(client, headers, subset_path, "giant")
|
||||
runner.check(
|
||||
bad_model.status_code == 400
|
||||
and bad_model.json()["error"]["code"] == "bad_request",
|
||||
"/train invalid model -> 400 bad_request",
|
||||
)
|
||||
missing_path = client.post(
|
||||
"/train", headers=headers, json={"path": "/tmp/no-such-pulsecast.csv", "model": "small"}
|
||||
)
|
||||
runner.check(missing_path.status_code == 400, "/train missing JSON path -> 400")
|
||||
|
||||
print("Small model: real CSV subset through the API")
|
||||
started = time.monotonic()
|
||||
submitted = _submit(client, headers, subset_path, "small")
|
||||
if submitted.status_code != 202:
|
||||
raise CheckFailed(f"/train small -> {submitted.status_code}: {submitted.text}")
|
||||
small_submitted = submitted.json()
|
||||
runner.check(
|
||||
small_submitted["status"] == "queued" and small_submitted["model"] == "small",
|
||||
f"/train small queued as job {small_submitted['job_id']}",
|
||||
)
|
||||
small_job = _wait_job(client, small_submitted["job_id"], headers)
|
||||
wall = round(time.monotonic() - started, 3)
|
||||
if small_job["status"] != "succeeded":
|
||||
raise CheckFailed(f"small job failed: {small_job}")
|
||||
small_metrics = small_job["metrics"]
|
||||
for key in ("accuracy", "precision", "recall", "f1", "roc_auc", "n_train", "n_test", "positive_rate"):
|
||||
if key not in small_metrics:
|
||||
raise CheckFailed(f"small metrics missing {key}: {small_metrics}")
|
||||
train_seconds = _seconds(small_job["started_at"], small_job["finished_at"])
|
||||
runner.check(
|
||||
small_job["model_version"].startswith("logit-"),
|
||||
f"small model_version {small_job['model_version']}",
|
||||
)
|
||||
runner.check(
|
||||
wall < 5.0,
|
||||
f"small job took {wall}s wall time including upload (contract < 5 s; "
|
||||
f"server timestamps have 1 s resolution, train_seconds={train_seconds})",
|
||||
)
|
||||
runner.check(small_metrics["accuracy"] > 0.6, f"small accuracy {small_metrics['accuracy']}")
|
||||
runner.check(
|
||||
(models_dir / "small_latest.joblib").exists(),
|
||||
"small model persisted to PULSECAST_MODELS",
|
||||
)
|
||||
meta_after_small = client.get("/meta").json()
|
||||
runner.check(
|
||||
meta_after_small["small_model"]["model_version"] == small_job["model_version"],
|
||||
"small model hot-swapped into /meta",
|
||||
)
|
||||
runner.check(
|
||||
meta_after_small["small_model"]["metrics"]["accuracy"] == small_metrics["accuracy"],
|
||||
"hot-swapped small metrics match the job",
|
||||
)
|
||||
predict_after_small = client.post(
|
||||
"/predict", json={"sensor": SENSORS[0], "horizon": "7d", "scenario": "base"}
|
||||
)
|
||||
runner.check(
|
||||
predict_after_small.status_code == 200
|
||||
and len(predict_after_small.json()["points"]) == 168,
|
||||
"/predict still works after the small swap",
|
||||
)
|
||||
report["jobs"] = {
|
||||
"small": {
|
||||
"job": small_job,
|
||||
"wall_seconds": wall,
|
||||
"train_seconds": train_seconds,
|
||||
"input": subset_info,
|
||||
}
|
||||
}
|
||||
|
||||
print("Forecast model: full CSV, hot-swap, 409 while busy")
|
||||
forecast_started = time.monotonic()
|
||||
forecast_response = _submit(client, headers, raw, "forecast")
|
||||
if forecast_response.status_code != 202:
|
||||
raise CheckFailed(
|
||||
f"/train forecast -> {forecast_response.status_code}: {forecast_response.text}"
|
||||
)
|
||||
forecast_submitted = forecast_response.json()
|
||||
runner.check(
|
||||
forecast_submitted["model"] == "forecast",
|
||||
f"/train forecast queued as job {forecast_submitted['job_id']}",
|
||||
)
|
||||
_wait_job(client, forecast_submitted["job_id"], headers, state="running", timeout=30)
|
||||
busy = _submit(client, headers, subset_path, "small")
|
||||
runner.check(
|
||||
busy.status_code == 409 and busy.json()["error"]["code"] == "conflict",
|
||||
"second /train while forecast is running -> 409 conflict",
|
||||
)
|
||||
forecast_job = _wait_job(client, forecast_submitted["job_id"], headers, timeout=300)
|
||||
forecast_wall = round(time.monotonic() - forecast_started, 3)
|
||||
if forecast_job["status"] != "succeeded":
|
||||
raise CheckFailed(f"forecast job failed: {forecast_job}")
|
||||
forecast_metrics = forecast_job["metrics"]
|
||||
for key in ("mae", "wape", "r2", "baseline_mae", "n_eval"):
|
||||
if key not in forecast_metrics:
|
||||
raise CheckFailed(f"forecast metrics missing {key}: {forecast_metrics}")
|
||||
runner.check(
|
||||
forecast_job["model_version"].startswith("histgb-"),
|
||||
f"forecast model_version {forecast_job['model_version']}",
|
||||
)
|
||||
runner.check(
|
||||
forecast_metrics["mae"] < forecast_metrics["baseline_mae"],
|
||||
f"retrained forecast MAE {forecast_metrics['mae']} < baseline {forecast_metrics['baseline_mae']}",
|
||||
)
|
||||
runner.check(
|
||||
(models_dir / "forecast_latest.joblib").exists(),
|
||||
"forecast model persisted to PULSECAST_MODELS",
|
||||
)
|
||||
meta_after_forecast = client.get("/meta").json()
|
||||
runner.check(
|
||||
meta_after_forecast["model_version"] == forecast_job["model_version"]
|
||||
and meta_after_forecast["model_source"] == "retrained",
|
||||
"forecast model hot-swapped into /meta as retrained",
|
||||
)
|
||||
predict_after_forecast = client.post(
|
||||
"/predict", json={"sensor": SENSORS[-1], "horizon": "48h", "scenario": "rainy-saturday"}
|
||||
)
|
||||
runner.check(
|
||||
predict_after_forecast.status_code == 200
|
||||
and predict_after_forecast.json()["model_version"] == forecast_job["model_version"],
|
||||
"/predict uses the retrained forecast model",
|
||||
)
|
||||
runner.check(client.get("/readyz").status_code == 200, "/readyz stays 200 after hot-swap")
|
||||
report["jobs"]["forecast"] = {
|
||||
"job": forecast_job,
|
||||
"wall_seconds": forecast_wall,
|
||||
"input": {"rows": "full challenge CSV", "bytes": raw.stat().st_size},
|
||||
}
|
||||
|
||||
print("Malformed CSV fails the job (not the process)")
|
||||
bad_csv = _submit(client, headers, garbage_path, "small")
|
||||
runner.check(bad_csv.status_code == 202, "malformed CSV is accepted as a job")
|
||||
bad_job = _wait_job(client, bad_csv.json()["job_id"], headers)
|
||||
runner.check(
|
||||
bad_job["status"] == "failed" and bool(bad_job["error"]),
|
||||
f"malformed CSV job failed with {bad_job['error']!r}",
|
||||
)
|
||||
runner.check(
|
||||
client.get("/healthz").status_code == 200,
|
||||
"/healthz still 200 after a failed job",
|
||||
)
|
||||
|
||||
report["versions_after_small"] = _versions(meta_after_small)
|
||||
report["versions_after_forecast"] = _versions(meta_after_forecast)
|
||||
report["checks"] = runner.checks
|
||||
report["ok"] = True
|
||||
finally:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
log_handle.close()
|
||||
if process.returncode not in (0, -15):
|
||||
print(f"server exited with {process.returncode}; log: {log_path}", file=sys.stderr)
|
||||
print(log_path.read_text(errors="replace")[-4000:], file=sys.stderr)
|
||||
report["server_log_tail"] = log_path.read_text(errors="replace")[-1000:]
|
||||
if args.keep:
|
||||
print(f"kept workdir {workdir}")
|
||||
else:
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
report["finished_at"] = _utc_now()
|
||||
return report
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--port", type=int, default=18000)
|
||||
parser.add_argument("--token", default=os.environ.get("CONTROL_TOKEN") or "e2e-local-token")
|
||||
parser.add_argument("--csv", default=str(config.raw_csv_path()))
|
||||
parser.add_argument("--days", type=int, default=60, help="days in the small-model CSV subset")
|
||||
parser.add_argument("--report", type=Path, help="write the JSON result here")
|
||||
parser.add_argument("--keep", action="store_true", help="keep the temporary workdir")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
runner = Runner()
|
||||
print(f"PulseCast end-to-end check on port {args.port}")
|
||||
started = time.monotonic()
|
||||
try:
|
||||
report = run(args, runner)
|
||||
except CheckFailed as exc:
|
||||
print(f"\nFAIL after {runner.checks} checks: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
elapsed = round(time.monotonic() - started, 1)
|
||||
report["elapsed_seconds"] = elapsed
|
||||
print(f"\nPASS: {runner.checks} checks in {elapsed}s")
|
||||
if args.report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n")
|
||||
print(f"report: {args.report}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user