"""`make data`: parse CSV, aggregate hourly, join weather/holidays, export JSON.""" from __future__ import annotations import argparse import json import logging import pandas as pd from . import config from .features import build_features, to_hourly from .parse import clean, load_raw, write_audit from .weather import ensure_weather log = logging.getLogger(__name__) def rfc3339(ts) -> str: return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ") def minute_iso(value: str | None) -> str | None: if not value: return None return pd.Timestamp(value).tz_convert("UTC").floor("min").strftime("%Y-%m-%dT%H:%M:%SZ") def _series_points(frame: pd.DataFrame) -> list[dict]: return [ {"t": rfc3339(t), "v": round(float(v), 2)} for t, v in zip(frame["t"], frame["v"], strict=True) ] def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]: multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)] battery_latest = ( battery.sort_values("time").groupby("sensor", observed=True).tail(1).set_index("sensor") ) out = [] for sensor in sorted(config.USABLE_SENSORS): rows = multi[multi["sensor"] == sensor] last = density[density["sensor"] == sensor]["time"].max() out.append( { "id": sensor, "name": config.DISPLAY_NAMES[sensor], "lat": round(float(rows["latitude"].median()), 6), "lon": round(float(rows["longitude"].median()), 6), "mean": round(float(rows["value"].mean()), 1), "p95": round(float(rows["value"].quantile(0.95)), 1), "battery": ( round(float(battery_latest.loc[sensor, "value"])) if sensor in battery_latest.index else None ), "last_seen": rfc3339(last), } ) return out def export_series(hourly: pd.DataFrame, data_version: str) -> dict: artifacts = config.artifacts_dir() series_dir = artifacts / "series" series_dir.mkdir(parents=True, exist_ok=True) per_sensor: dict[str, list[dict]] = {} for sensor in sorted(config.USABLE_SENSORS): frame = hourly[hourly["sensor"] == sensor] pts = _series_points(frame) per_sensor[sensor] = pts (series_dir / f"{sensor}.json").write_text( json.dumps( {"sensor": sensor, "step": "1h", "points": pts}, ensure_ascii=False ) + "\n" ) payload = {"step": "1h", "data_version": data_version, "series": per_sensor} (artifacts / "series.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n") return payload def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict: current = meta.get("current") or {} now_t = current.get("t") if now_t: ts = pd.Timestamp(now_t, tz="UTC") if pd.Timestamp(now_t).tz is None else pd.Timestamp(now_t).tz_convert("UTC") else: ts = pd.Timestamp(meta["fetched_at"]).tz_convert("UTC") row = forecast.iloc[(forecast["time"] - ts).abs().argmin()] return { "t": rfc3339(ts), "temp_c": round(float(current.get("temp_c", row["temp_c"])), 1), "precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2), "cloud_pct": round(float(current.get("cloud_pct", row["cloud_pct"]))), "wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1), "humidity_pct": round(float(current.get("humidity_pct", row["humidity_pct"]))), } def export_context(forecast: pd.DataFrame, meta: dict) -> dict: now = _weather_now(forecast, meta) start = pd.Timestamp(now["t"]).floor("h") window = forecast[(forecast["time"] >= start) & (forecast["time"] < start + pd.Timedelta(days=7))] points = [ { "t": rfc3339(row.time), "temp_c": round(float(row.temp_c), 1), "precip_mm": round(float(row.precip_mm), 2), "cloud_pct": round(float(row.cloud_pct)), "wind_kmh": round(float(row.wind_kmh), 1), "humidity_pct": round(float(row.humidity_pct)), } for row in window.itertuples(index=False) ] holidays = [ {"date": date, "name": name} for date, name in sorted(config.HOLIDAYS.items()) ] payload = { "weather": {"now": now, "forecast": points}, "holidays": holidays, "updated_at": meta["fetched_at"], "cached": False, } (config.artifacts_dir() / "context.json").write_text( json.dumps(payload, indent=2, ensure_ascii=False) + "\n" ) return payload def run_data(*, refresh_weather: bool = False) -> dict: logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") artifacts = config.artifacts_dir() cache = config.cache_dir() artifacts.mkdir(parents=True, exist_ok=True) cache.mkdir(parents=True, exist_ok=True) raw = load_raw() density, battery, audit = clean(raw) write_audit(audit) # The product window is the 8-sensor period. density-01 has 13 months of # history, but using it alone would add single-sensor annual seasonality that # the other seven locations cannot support. Keep the full-file audit above. window_start = pd.Timestamp(config.MULTI_SENSOR_START) density = density[density["time"] >= window_start].copy() battery = battery[battery["time"] >= window_start].copy() density[["time", "sensor", "sensor_id", "value"]].to_parquet( cache / "density_clean.parquet", index=False ) battery[["time", "sensor", "sensor_id", "value"]].to_parquet( cache / "battery_clean.parquet", index=False ) log.info( "parsed %s rows; %s usable sensors; %s density rows in the 8-sensor window", audit["raw_rows"], audit["usable_sensors"], len(density), ) hourly = to_hourly(density) hourly.to_parquet(cache / "hourly.parquet", index=False) archive, forecast, meta = ensure_weather( density["time"].min(), density["time"].max(), refresh=refresh_weather ) features = build_features(hourly, archive) features.to_parquet(cache / "hourly_features.parquet", index=False) log.info("built %s hourly feature rows", len(features)) # Bake compact weather frames beside the JSON so the FastAPI image can # retrain and serve /predict without the git-ignored data/cache directory. archive.to_parquet(artifacts / "weather_archive.parquet", index=False) forecast.to_parquet(artifacts / "weather_forecast.parquet", index=False) (artifacts / "weather_meta.json").write_text( json.dumps(meta, indent=2, ensure_ascii=False) + "\n" ) data_version = minute_iso(audit["time_range"]["to"]) sensors = export_sensors(density, battery) (artifacts / "sensors.json").write_text( json.dumps(sensors, indent=2, ensure_ascii=False) + "\n" ) series = export_series(hourly, data_version) context = export_context(forecast, meta) summary = { "sensors": len(sensors), "series_sensors": len(series["series"]), "hourly_rows": len(hourly), "feature_rows": len(features), "weather_rows": len(archive), "context_forecast_hours": len(context["weather"]["forecast"]), "data_version": data_version, "audit_ok": audit["all_checks_ok"], } log.info("data artifacts written to %s: %s", artifacts, json.dumps(summary)) return summary def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--refresh-weather", action="store_true") args = parser.parse_args() run_data(refresh_weather=args.refresh_weather) if __name__ == "__main__": main()