- finish the global HistGB bundle, scenario forecasts and 28-day ablation - export sensors/series/forecasts/metrics/context plus baked weather frames - add FastAPI health/meta/predict/simulate and background /train jobs - add the p75 busy classifier, retraining from CSV, and hot-swap persistence - add pytest coverage, multi-stage non-root Dockerfile, README and model card Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
212 lines
7.7 KiB
Python
212 lines
7.7 KiB
Python
"""`make data`: parse CSV, aggregate hourly, join weather/holidays, export JSON."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
|
|
import pandas as pd
|
|
|
|
from . import config
|
|
from .features import build_features, to_hourly
|
|
from .parse import clean, load_raw, write_audit
|
|
from .weather import ensure_weather
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def rfc3339(ts) -> str:
|
|
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def minute_iso(value: str | None) -> str | None:
|
|
if not value:
|
|
return None
|
|
return pd.Timestamp(value).tz_convert("UTC").floor("min").strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def _series_points(frame: pd.DataFrame) -> list[dict]:
|
|
return [
|
|
{"t": rfc3339(t), "v": round(float(v), 2)}
|
|
for t, v in zip(frame["t"], frame["v"], strict=True)
|
|
]
|
|
|
|
|
|
def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
|
|
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
|
|
battery_latest = (
|
|
battery.sort_values("time").groupby("sensor", observed=True).tail(1).set_index("sensor")
|
|
)
|
|
out = []
|
|
for sensor in sorted(config.USABLE_SENSORS):
|
|
rows = multi[multi["sensor"] == sensor]
|
|
last = density[density["sensor"] == sensor]["time"].max()
|
|
out.append(
|
|
{
|
|
"id": sensor,
|
|
"name": config.DISPLAY_NAMES[sensor],
|
|
"lat": round(float(rows["latitude"].median()), 6),
|
|
"lon": round(float(rows["longitude"].median()), 6),
|
|
"mean": round(float(rows["value"].mean()), 1),
|
|
"p95": round(float(rows["value"].quantile(0.95)), 1),
|
|
"battery": (
|
|
round(float(battery_latest.loc[sensor, "value"]))
|
|
if sensor in battery_latest.index
|
|
else None
|
|
),
|
|
"last_seen": rfc3339(last),
|
|
}
|
|
)
|
|
return out
|
|
|
|
|
|
def export_series(hourly: pd.DataFrame, data_version: str) -> dict:
|
|
artifacts = config.artifacts_dir()
|
|
series_dir = artifacts / "series"
|
|
series_dir.mkdir(parents=True, exist_ok=True)
|
|
per_sensor: dict[str, list[dict]] = {}
|
|
for sensor in sorted(config.USABLE_SENSORS):
|
|
frame = hourly[hourly["sensor"] == sensor]
|
|
pts = _series_points(frame)
|
|
per_sensor[sensor] = pts
|
|
(series_dir / f"{sensor}.json").write_text(
|
|
json.dumps(
|
|
{"sensor": sensor, "step": "1h", "points": pts}, ensure_ascii=False
|
|
)
|
|
+ "\n"
|
|
)
|
|
payload = {"step": "1h", "data_version": data_version, "series": per_sensor}
|
|
(artifacts / "series.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n")
|
|
return payload
|
|
|
|
|
|
def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
|
|
current = meta.get("current") or {}
|
|
now_t = current.get("t")
|
|
if now_t:
|
|
ts = pd.Timestamp(now_t, tz="UTC") if pd.Timestamp(now_t).tz is None else pd.Timestamp(now_t).tz_convert("UTC")
|
|
else:
|
|
ts = pd.Timestamp(meta["fetched_at"]).tz_convert("UTC")
|
|
row = forecast.iloc[(forecast["time"] - ts).abs().argmin()]
|
|
return {
|
|
"t": rfc3339(ts),
|
|
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
|
|
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
|
|
"cloud_pct": round(float(current.get("cloud_pct", row["cloud_pct"]))),
|
|
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
|
|
"humidity_pct": round(float(current.get("humidity_pct", row["humidity_pct"]))),
|
|
}
|
|
|
|
|
|
def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
|
|
now = _weather_now(forecast, meta)
|
|
start = pd.Timestamp(now["t"]).floor("h")
|
|
window = forecast[(forecast["time"] >= start) & (forecast["time"] < start + pd.Timedelta(days=7))]
|
|
points = [
|
|
{
|
|
"t": rfc3339(row.time),
|
|
"temp_c": round(float(row.temp_c), 1),
|
|
"precip_mm": round(float(row.precip_mm), 2),
|
|
"cloud_pct": round(float(row.cloud_pct)),
|
|
"wind_kmh": round(float(row.wind_kmh), 1),
|
|
"humidity_pct": round(float(row.humidity_pct)),
|
|
}
|
|
for row in window.itertuples(index=False)
|
|
]
|
|
holidays = [
|
|
{"date": date, "name": name} for date, name in sorted(config.HOLIDAYS.items())
|
|
]
|
|
payload = {
|
|
"weather": {"now": now, "forecast": points},
|
|
"holidays": holidays,
|
|
"updated_at": meta["fetched_at"],
|
|
"cached": False,
|
|
}
|
|
(config.artifacts_dir() / "context.json").write_text(
|
|
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
|
)
|
|
return payload
|
|
|
|
|
|
def run_data(*, refresh_weather: bool = False) -> dict:
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
|
artifacts = config.artifacts_dir()
|
|
cache = config.cache_dir()
|
|
artifacts.mkdir(parents=True, exist_ok=True)
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
|
|
raw = load_raw()
|
|
density, battery, audit = clean(raw)
|
|
write_audit(audit)
|
|
|
|
# The product window is the 8-sensor period. density-01 has 13 months of
|
|
# history, but using it alone would add single-sensor annual seasonality that
|
|
# the other seven locations cannot support. Keep the full-file audit above.
|
|
window_start = pd.Timestamp(config.MULTI_SENSOR_START)
|
|
density = density[density["time"] >= window_start].copy()
|
|
battery = battery[battery["time"] >= window_start].copy()
|
|
|
|
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
|
cache / "density_clean.parquet", index=False
|
|
)
|
|
battery[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
|
cache / "battery_clean.parquet", index=False
|
|
)
|
|
log.info(
|
|
"parsed %s rows; %s usable sensors; %s density rows in the 8-sensor window",
|
|
audit["raw_rows"],
|
|
audit["usable_sensors"],
|
|
len(density),
|
|
)
|
|
|
|
hourly = to_hourly(density)
|
|
hourly.to_parquet(cache / "hourly.parquet", index=False)
|
|
|
|
archive, forecast, meta = ensure_weather(
|
|
density["time"].min(), density["time"].max(), refresh=refresh_weather
|
|
)
|
|
features = build_features(hourly, archive)
|
|
features.to_parquet(cache / "hourly_features.parquet", index=False)
|
|
log.info("built %s hourly feature rows", len(features))
|
|
|
|
# Bake compact weather frames beside the JSON so the FastAPI image can
|
|
# retrain and serve /predict without the git-ignored data/cache directory.
|
|
archive.to_parquet(artifacts / "weather_archive.parquet", index=False)
|
|
forecast.to_parquet(artifacts / "weather_forecast.parquet", index=False)
|
|
(artifacts / "weather_meta.json").write_text(
|
|
json.dumps(meta, indent=2, ensure_ascii=False) + "\n"
|
|
)
|
|
|
|
data_version = minute_iso(audit["time_range"]["to"])
|
|
sensors = export_sensors(density, battery)
|
|
(artifacts / "sensors.json").write_text(
|
|
json.dumps(sensors, indent=2, ensure_ascii=False) + "\n"
|
|
)
|
|
series = export_series(hourly, data_version)
|
|
context = export_context(forecast, meta)
|
|
|
|
summary = {
|
|
"sensors": len(sensors),
|
|
"series_sensors": len(series["series"]),
|
|
"hourly_rows": len(hourly),
|
|
"feature_rows": len(features),
|
|
"weather_rows": len(archive),
|
|
"context_forecast_hours": len(context["weather"]["forecast"]),
|
|
"data_version": data_version,
|
|
"audit_ok": audit["all_checks_ok"],
|
|
}
|
|
log.info("data artifacts written to %s: %s", artifacts, json.dumps(summary))
|
|
return summary
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--refresh-weather", action="store_true")
|
|
args = parser.parse_args()
|
|
run_data(refresh_weather=args.refresh_weather)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|