docs: record 120-client SSE load, train and fallback results
Add staffing timing to the integration harness and commit the verified run: small train 171ms, forecast train 3.6s, SSE propagation 15ms, 120/120 clients for 300s with 0 drops and p95 5.6ms, 20 heartbeats at 15s, FastAPI-down fallback from the snapshot, and recovery in 1.6s.
This commit is contained in:
1 parent
ac7ceca7ec
commit
b3146e0862
15 files changed
+481
-1
No files matched your search
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"fastapi_down": {
|
||||
"cache_failures": 1,
|
||||
"last_error": "ml: /bootstrap: Get \"http://127.0.0.1:18000/bootstrap\": dial tcp 127.0.0.1:18000: connect: connection refused",
|
||||
"next_retry_at": "2026-11-14T05:34:31Z",
|
||||
"last_success_at": "2026-11-14T05:34:20Z",
|
||||
"forecast_cached": true,
|
||||
"staffing_cached": true,
|
||||
"readyz": 200,
|
||||
"sensors": 8
|
||||
},
|
||||
"recovery": {
|
||||
"after_ms": 1622,
|
||||
"last_success_before": "2026-11-14T05:34:20Z",
|
||||
"last_success_after": "2026-11-14T05:34:33Z",
|
||||
"forecast_cached": false,
|
||||
"failures": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"sensor":"hn-stadtgalerie-01","horizon":"7d","scenario":"base","points":[{"t":"2026-11-14T06:00:00Z","v":4.35,"lo":0,"hi":9.23},{"t":"2026-11-14T07:00:00Z","v":8.07,"lo":3.25,"hi":12.95},{"t":"2026-11-14T08:00:00Z","v":23.45,"lo":18.63,"hi":28.33},{"t":"2026-11-14T09:00:00Z","v":35.8,"lo":30.98,"hi":40.68},{"t":"2026-11-14T10:00:00Z","v":67.45,"lo":62.62,"hi":72.32},{"t":"2026-11-14T11:00:00Z","v":86.07,"lo":81.25,"hi":90.95},{"t":"2026-11-14T12:00:00Z","v":88.9,"lo":84.08,"hi":93.78},{"t":"2026-11-14T13:00:00Z","v":87.06,"lo":82.23,"hi":91.94},{"t":"2026-11-14T14:00:00Z","v":88.1,"lo":83.27,"hi":92.97},{"t":"2026-11-14T15:00:00Z","v":84.2,"lo":79.37,"hi":89.07},{"t":"2026-11-14T16:00:00Z","v":85.68,"lo":80.86,"hi":90.56},{"t":"2026-11-14T17:00:00Z","v":76.15,"lo":71.33,"hi":81.03},{"t":"2026-11-14T18:00:00Z","v":45.34,"lo":40.51,"hi":50.21},{"t":"2026-11-14T19:00:00Z","v":20.09,"lo":15.26,"hi":24.97},{"t":"2026-11-14T20:00:00Z","v":13,"lo":8.17,"hi":17.87},{"t":"2026-11-14T21:00:00Z","v":7.36,"lo":2.54,"hi":12.24},{"t":"2026-11-14T22:00:00Z","v":5.2,"lo":0.37,"hi":10.07},{"t":"2026-11-14T23:00:00Z","v":4.64,"lo":0,"hi":9.52},{"t":"2026-11-15T00:00:00Z","v":3.25,"lo":0,"hi":8.13},{"t":"2026-11-15T01:00:00Z","v":2.35,"lo":0,"hi":7.22},{"t":"2026-11-15T02:00:00Z","v":2.46,"lo":0,"hi":7.34},{"t":"2026-11-15T03:00:00Z","v":1.78,"lo":0,"hi":6.66},{"t":"2026-11-15T04:00:00Z","v":1.78,"lo":0,"hi":6.65},{"t":"2026-11-15T05:00:00Z","v":1.77,"lo":0,"hi":6.65},{"t":"2026-11-15T06:00:00Z","v":2.16,"lo":0,"hi":7.04},{"t":"2026-11-15T07:00:00Z","v":2.71,"lo":0,"hi":7.59},{"t":"2026-11-15T08:00:00Z","v":4.92,"lo":0.09,"hi":9.79},{"t":"2026-11-15T09:00:00Z","v":6.51,"lo":1.68,"hi":11.39},{"t":"2026-11-15T10:00:00Z","v":6.41,"lo":1.58,"hi":11.28},{"t":"2026-11-15T11:00:00Z","v":12.24,"lo":7.41,"hi":17.11},{"t":"2026-11-15T12:00:00Z","v":17.2,"lo":12.38,"hi":22.08},{"t":"2026-11-15T13:00:00Z","v":17.44,"lo":12.61,"hi":22.31},{"t":"2026-11-15T14:00:00Z","v":17.05,"lo":12.23,"hi":21.93},{"t":"2026-11-15T15:00:00Z","v":15.94,"lo":11.12,"hi":20.82},{"t":"2026-11-15T16:00:00Z","v":16.44,"lo":11.61,"hi":21.32},{"t":"2026-11-15T17:00:00Z","v":14.1,"lo":9.27,"hi":18.97},{"t":"2026-11-15T18:00:00Z","v":8.92,"lo":4.1,"hi":13.8},{"t":"2026-11-15T19:00:00Z","v":4.78,"lo":0,"hi":9.65},{"t":"2026-11-15T20:00:00Z","v":2.49,"lo":0,"hi":7.36},{"t":"2026-11-15T21:00:00Z","v":1.6,"lo":0,"hi":6.47},{"t":"2026-11-15T22:00:00Z","v":1.84,"lo":0,"hi":6.72},{"t":"2026-11-15T23:00:00Z","v":1.69,"lo":0,"hi":6.57},{"t":"2026-11-16T00:00:00Z","v":1.59,"lo":0,"hi":6.47},{"t":"2026-11-16T01:00:00Z","v":1.57,"lo":0,"hi":6.45},{"t":"2026-11-16T02:00:00Z","v":1.57,"lo":0,"hi":6.45},{"t":"2026-11-16T03:00:00Z","v":1.57,"lo":0,"hi":6.45},{"t":"2026-11-16T04:00:00Z","v":1.61,"lo":0,"hi":6.48},{"t":"2026-11-16T05:00:00Z","v":2.59,"lo":0,"hi":7.46},{"t":"2026-11-16T06:00:00Z","v":7.05,"lo":2.22,"hi":11.93},{"t":"2026-11-16T07:00:00Z","v":14.22,"lo":9.39,"hi":19.09},{"t":"2026-11-16T08:00:00Z","v":18.83,"lo":14.01,"hi":23.71},{"t":"2026-11-16T09:00:00Z","v":28.14,"lo":23.31,"hi":33.01},{"t":"2026-11-16T10:00:00Z","v":39.14,"lo":34.31,"hi":44.01},{"t":"2026-11-16T11:00:00Z","v":51.34,"lo":46.52,"hi":56.22},{"t":"2026-11-16T12:00:00Z","v":50.18,"lo":45.35,"hi":55.05},{"t":"2026-11-16T13:00:00Z","v":50.22,"lo":45.4,"hi":55.1},{"t":"2026-11-16T14:00:00Z","v":50.3,"lo":45.47,"hi":55.17},{"t":"2026-11-16T15:00:00Z","v":49.68,"lo":44.85,"hi":54.56},{"t":"2026-11-16T16:00:00Z","v":51.29,"lo":46.46,"hi":56.16},{"t":"2026-11-16T17:00:00Z","v":43.09,"lo":38.26,"hi":47.96},{"t":"2026-11-16T18:00:00Z","v":21.34,"lo":16.51,"hi":26.22},{"t":"2026-11-16T19:00:00Z","v":11.95,"lo":7.12,"hi":16.82},{"t":"2026-11-16T20:00:00Z","v":5.95,"lo":1.13,"hi":10.83},{"t":"2026-11-16T21:00:00Z","v":2.98,"lo":0,"hi":7.85},{"t":"2026-11-16T22:00:00Z","v":2.19,"lo":0,"hi":7.07},{"t":"2026-11-16T23:00:00Z","v":2,"lo":0,"hi":6.87},{"t":"2026-11-17T00:00:00Z","v":1.92,"lo":0,"hi":6.79},{"t":"2026-11-17T01:00:00Z","v":1.68,"lo":0,"hi":6.55},{"t":"2026-11-17T02:00:00Z","v":1.59,"lo":0,"hi":6.46},{"t":"2026-11-17T03:00:00Z","v":1.59,"lo":0,"hi":6.46},{"t":"2026-11-17T04:00:00Z","v":2.4,"lo":0,"hi":7.28},{"t":"2026-11-17T05:00:00Z","v":3.38,"lo":0,"hi":8.25},{"t":"2026-11-17T06:00:00Z","v":8.12,"lo":3.29,"hi":12.99},{"t":"2026-11-17T07:00:00Z","v":16.28,"lo":11.45,"hi":21.16},{"t":"2026-11-17T08:00:00Z","v":17.6,"lo":12.77,"hi":22.47},{"t":"2026-11-17T09:00:00Z","v":27.47,"lo":22.65,"hi":32.35},{"t":"2026-11-17T10:00:00Z","v":35.28,"lo":30.45,"hi":40.15},{"t":"2026-11-17T11:00:00Z","v":50.47,"lo":45.64,"hi":55.34},{"t":"2026-11-17T12:00:00Z","v":48.11,"lo":43.28,"hi":52.98},{"t":"2026-11-17T13:00:00Z","v":42.24,"lo":37.41,"hi":47.11},{"t":"2026-11-17T14:00:00Z","v":49.05,"lo":44.22,"hi":53.92},{"t":"2026-11-17T15:00:00Z","v":48.19,"lo":43.37,"hi":53.07},{"t":"2026-11-17T16:00:00Z","v":47.31,"lo":42.48,"hi":52.18},{"t":"2026-11-17T17:00:00Z","v":32.93,"lo":28.1,"hi":37.8},{"t":"2026-11-17T18:00:00Z","vLine truncated
|
||||
@@ -0,0 +1,131 @@
|
||||
{
|
||||
"started_at": "2026-11-14T05:34:33.422221537Z",
|
||||
"duration_s": 300.008,
|
||||
"base_url": "http://127.0.0.1:18080",
|
||||
"clients_configured": 120,
|
||||
"clients_peak_connected": 120,
|
||||
"clients_connected_after_warmup": 120,
|
||||
"connection_drops": 0,
|
||||
"scenario_post_errors": 0,
|
||||
"scenario_changes": 9,
|
||||
"scenario_events_received": 1080,
|
||||
"scenario_events_missing": 0,
|
||||
"scenario_latency_samples": 1080,
|
||||
"scenario_p50_ms": 1.3,
|
||||
"scenario_p95_ms": 5.6,
|
||||
"scenario_p99_ms": 5.8,
|
||||
"scenario_max_ms": 5.812,
|
||||
"changes": [
|
||||
{
|
||||
"version": 3,
|
||||
"post_at": "2026-11-14T06:35:03.474811874+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 5.5,
|
||||
"p95_ms": 5.8,
|
||||
"p99_ms": 5.8,
|
||||
"max_ms": 5.812
|
||||
},
|
||||
{
|
||||
"version": 6,
|
||||
"post_at": "2026-11-14T06:35:33.474996501+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 2.4,
|
||||
"p95_ms": 2.7,
|
||||
"p99_ms": 2.8,
|
||||
"max_ms": 2.791
|
||||
},
|
||||
{
|
||||
"version": 7,
|
||||
"post_at": "2026-11-14T06:36:03.474746595+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 1.1,
|
||||
"p95_ms": 1.3,
|
||||
"p99_ms": 1.4,
|
||||
"max_ms": 1.353
|
||||
},
|
||||
{
|
||||
"version": 8,
|
||||
"post_at": "2026-11-14T06:36:33.467364475+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 1.7,
|
||||
"p95_ms": 2.4,
|
||||
"p99_ms": 2.4,
|
||||
"max_ms": 2.396
|
||||
},
|
||||
{
|
||||
"version": 9,
|
||||
"post_at": "2026-11-14T06:37:03.472992832+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 1,
|
||||
"p95_ms": 1.3,
|
||||
"p99_ms": 1.3,
|
||||
"max_ms": 1.342
|
||||
},
|
||||
{
|
||||
"version": 10,
|
||||
"post_at": "2026-11-14T06:37:33.475595017+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 1.1,
|
||||
"p95_ms": 1.3,
|
||||
"p99_ms": 1.4,
|
||||
"max_ms": 1.434
|
||||
},
|
||||
{
|
||||
"version": 11,
|
||||
"post_at": "2026-11-14T06:38:03.474743535+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 1.3,
|
||||
"p95_ms": 1.4,
|
||||
"p99_ms": 1.4,
|
||||
"max_ms": 1.375
|
||||
},
|
||||
{
|
||||
"version": 12,
|
||||
"post_at": "2026-11-14T06:38:33.469461499+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 1.5,
|
||||
"p95_ms": 1.7,
|
||||
"p99_ms": 1.7,
|
||||
"max_ms": 1.718
|
||||
},
|
||||
{
|
||||
"version": 13,
|
||||
"post_at": "2026-11-14T06:39:03.475695318+01:00",
|
||||
"connected_at_post": 120,
|
||||
"received": 120,
|
||||
"missing": 0,
|
||||
"p50_ms": 1,
|
||||
"p95_ms": 1.2,
|
||||
"p99_ms": 1.2,
|
||||
"max_ms": 1.249
|
||||
}
|
||||
],
|
||||
"heartbeat": {
|
||||
"unique_server_timestamps": 20,
|
||||
"min_interval_ms": 15000,
|
||||
"median_interval_ms": 15000,
|
||||
"p95_interval_ms": 15000,
|
||||
"max_interval_ms": 15000,
|
||||
"per_client_min": 20,
|
||||
"per_client_median": 20,
|
||||
"per_client_max": 20
|
||||
},
|
||||
"errors": null,
|
||||
"passed": true
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
clients connected: 120/120 after warmup
|
||||
scenario v3: 120/120 clients p50=5.5ms p95=5.8ms max=5.8ms
|
||||
scenario v6: 120/120 clients p50=2.4ms p95=2.7ms max=2.8ms
|
||||
scenario v7: 120/120 clients p50=1.1ms p95=1.3ms max=1.4ms
|
||||
scenario v8: 120/120 clients p50=1.7ms p95=2.4ms max=2.4ms
|
||||
scenario v9: 120/120 clients p50=1.0ms p95=1.3ms max=1.3ms
|
||||
scenario v10: 120/120 clients p50=1.1ms p95=1.3ms max=1.4ms
|
||||
scenario v11: 120/120 clients p50=1.3ms p95=1.4ms max=1.4ms
|
||||
scenario v12: 120/120 clients p50=1.5ms p95=1.7ms max=1.7ms
|
||||
scenario v13: 120/120 clients p50=1.0ms p95=1.2ms max=1.2ms
|
||||
|
||||
=== SSE load test ===
|
||||
duration: 300.0s
|
||||
clients: 120 configured, 120 peak connected, 120 after warmup
|
||||
connections: 0 drops/reconnects, 0 scenario POST errors
|
||||
scenario changes: 9 (1080 events received, 0 missing)
|
||||
scenario latency: p50=1.3ms p95=5.6ms p99=5.8ms max=5.8ms (n=1080)
|
||||
heartbeat: 20 server pings, interval min=15000ms median=15000ms p95=15000ms max=15000ms
|
||||
heartbeat/client: min=20 median=20 max=20
|
||||
result: PASS
|
||||
json written to /home/noel/Documents/Hackathon_2026/api-go.git/feat-integration/docs/results/load.json
|
||||
@@ -0,0 +1 @@
|
||||
{"data_version":"2026-11-07T23:51:00Z","model_version":"histgb-2026-11-14","context_version":"2026-11-14T03:47:09Z","coverage":{"sensors":8,"from":"2026-06-12T00:00:00Z","to":"2026-11-07T23:51:00Z","cadence_min":15},"metrics":{"model":"weather-7d","mae":3.32,"wape":27.5,"r2":0.777,"baseline_mae":5.4,"baseline_wape":44.7},"limits":["estimated devices, not people","8 sensors in a ~580 m inner-city cluster","5 months of multi-sensor data; no annual seasonality claim","observed weather in training, forecast weather in scenarios","holidays are context, not a large accuracy claim"],"cache":{"source":"fastapi","last_success_at":"2026-11-14T05:34:09Z","last_error":null,"next_retry_at":null,"failures":0}}
|
||||
@@ -0,0 +1,13 @@
|
||||
retry: 3000
|
||||
|
||||
event: scenario
|
||||
id: 1
|
||||
data: {"version":1,"scenario":{"scenario":"base","weather":{"temp_delta_c":0,"precip_mm":0},"event":{"type":"","uplift":0}},"at":"2026-11-14T05:34:09Z"}
|
||||
|
||||
event: scenario
|
||||
id: 2
|
||||
data: {"version":2,"scenario":{"scenario":"sunny-saturday-market","weather":{"temp_delta_c":4,"precip_mm":0},"event":{"type":"market","uplift":0.2}},"at":"2026-11-14T05:34:20Z"}
|
||||
|
||||
event: ping
|
||||
data: {"t":"2026-11-14T05:34:24Z"}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"scenario_version": 2,
|
||||
"propagation_ms": 15,
|
||||
"heartbeat_after_ms": 4260,
|
||||
"heartbeat_interval_s": 15,
|
||||
"retry_ms": 3000,
|
||||
"reconnect_ms": 18,
|
||||
"reconnect_current_version": 2
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cold_ms": 19,
|
||||
"warm_ms": 0,
|
||||
"hours": 70,
|
||||
"shifts": 21,
|
||||
"coverage_pct": 100,
|
||||
"cost_eur": 3360,
|
||||
"cached": false,
|
||||
"note": "follow-up run on ports 18100/18180; same commit"
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
PulseCast Go integration 2026-11-14T05:39:33Z
|
||||
FastAPI http://127.0.0.1:18000, Go http://127.0.0.1:18080
|
||||
small CSV: 24636 rows (2026-09-01 .. 2026-10-16)
|
||||
small job: status=succeeded version=logit-20261114-053410 accuracy=0.824 roc_auc=0.849 submit_to_terminal_ms=171
|
||||
SSE: {"scenario_version":2,"propagation_ms":15,"heartbeat_after_ms":4260,"heartbeat_interval_s":15,"retry_ms":3000,"reconnect_ms":18,"reconnect_current_version":2}
|
||||
fallback: {"fastapi_down":{"cache_failures":1,"last_error":"ml: /bootstrap: Get \"http://127.0.0.1:18000/bootstrap\": dial tcp 127.0.0.1:18000: connect: connection refused","next_retry_at":"2026-11-14T05:34:31Z","last_success_at":"2026-11-14T05:34:20Z","forecast_cached":true,"staffing_cached":true,"readyz":200,"sensors":8},"recovery":{"after_ms":1622,"last_success_before":"2026-11-14T05:34:20Z","last_success_after":"2026-11-14T05:34:33Z","forecast_cached":false,"failures":0}}
|
||||
forecast job: status=succeeded version=histgb-20261114-053413 mae=4.83 wape=33.0 submit_to_terminal_ms=3571
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"model": "forecast",
|
||||
"submit_http": 202,
|
||||
"submit_ms": 15,
|
||||
"submit_to_terminal_ms": 3571,
|
||||
"job": {
|
||||
"job_id": "2f4e7c5a5bc14abd81d4eb5cdbeb0e1a",
|
||||
"status": "succeeded",
|
||||
"model": "forecast",
|
||||
"submitted_at": "2026-11-14T05:34:10Z",
|
||||
"started_at": "2026-11-14T05:34:10Z",
|
||||
"finished_at": "2026-11-14T05:34:13Z",
|
||||
"metrics": {
|
||||
"model": "weather-7d",
|
||||
"mae": 4.83,
|
||||
"wape": 33.0,
|
||||
"r2": 0.703,
|
||||
"baseline": "seasonal_naive",
|
||||
"baseline_mae": 6.8,
|
||||
"baseline_wape": 46.5,
|
||||
"baseline_r2": 0.318,
|
||||
"n_eval": 4919,
|
||||
"ablation": [
|
||||
{
|
||||
"name": "seasonal_naive",
|
||||
"description": "exact 168-row lag (same observed hour last week)",
|
||||
"mae": 6.8,
|
||||
"wape": 46.5,
|
||||
"r2": 0.318,
|
||||
"n": 4919
|
||||
},
|
||||
{
|
||||
"name": "day_no_weather",
|
||||
"description": "calendar + lags 24/48/168 + 24h/168h rolling means",
|
||||
"mae": 5.5,
|
||||
"wape": 37.6,
|
||||
"r2": 0.623,
|
||||
"n": 4919
|
||||
},
|
||||
{
|
||||
"name": "day_holiday",
|
||||
"description": "day-ahead + Baden-Württemberg holiday flag",
|
||||
"mae": 5.5,
|
||||
"wape": 37.6,
|
||||
"r2": 0.623,
|
||||
"n": 4919
|
||||
},
|
||||
{
|
||||
"name": "day_weather",
|
||||
"description": "day-ahead + temperature + rain (observed weather)",
|
||||
"mae": 5.03,
|
||||
"wape": 34.4,
|
||||
"r2": 0.682,
|
||||
"n": 4919
|
||||
},
|
||||
{
|
||||
"name": "week_no_weather",
|
||||
"description": "7-day-safe calendar + previous same weekday/hour values",
|
||||
"mae": 5.4,
|
||||
"wape": 36.9,
|
||||
"r2": 0.631,
|
||||
"n": 4919
|
||||
},
|
||||
{
|
||||
"name": "week_weather",
|
||||
"description": "7-day-safe + temperature + rain (production model)",
|
||||
"mae": 4.83,
|
||||
"wape": 33.0,
|
||||
"r2": 0.703,
|
||||
"n": 4919
|
||||
}
|
||||
]
|
||||
},
|
||||
"model_version": "histgb-20261114-053413",
|
||||
"error": null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"model": "small",
|
||||
"submit_http": 202,
|
||||
"submit_ms": 25,
|
||||
"submit_to_terminal_ms": 171,
|
||||
"job": {
|
||||
"job_id": "9b6112d7baa34ee8ad236db0d77b836b",
|
||||
"status": "succeeded",
|
||||
"model": "small",
|
||||
"submitted_at": "2026-11-14T05:34:10Z",
|
||||
"started_at": "2026-11-14T05:34:10Z",
|
||||
"finished_at": "2026-11-14T05:34:10Z",
|
||||
"metrics": {
|
||||
"accuracy": 0.824,
|
||||
"precision": 0.589,
|
||||
"recall": 0.42,
|
||||
"f1": 0.49,
|
||||
"roc_auc": 0.849,
|
||||
"n_train": 5363,
|
||||
"n_test": 1728,
|
||||
"positive_rate": 0.201,
|
||||
"sensors": 8
|
||||
},
|
||||
"model_version": "logit-20261114-053410",
|
||||
"error": null
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user