feat: add pipeline and training entrypoints
This commit is contained in:
1 parent
8a81af771e
commit
68402803b2
2 files changed
+465
No files matched your search
@@ -0,0 +1,196 @@
|
||||
"""`make data`: parse CSV, aggregate hourly, join weather/holidays, export JSON."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from . import config
|
||||
from .features import build_features, to_hourly
|
||||
from .parse import clean, load_raw, write_audit
|
||||
from .weather import ensure_weather
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rfc3339(ts) -> str:
|
||||
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def minute_iso(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
return pd.Timestamp(value).tz_convert("UTC").floor("min").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _series_points(frame: pd.DataFrame) -> list[dict]:
|
||||
return [
|
||||
{"t": rfc3339(t), "v": round(float(v), 2)}
|
||||
for t, v in zip(frame["t"], frame["v"], strict=True)
|
||||
]
|
||||
|
||||
|
||||
def export_sensors(density: pd.DataFrame, battery: pd.DataFrame) -> list[dict]:
|
||||
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
|
||||
battery_latest = (
|
||||
battery.sort_values("time").groupby("sensor", observed=True).tail(1).set_index("sensor")
|
||||
)
|
||||
out = []
|
||||
for sensor in sorted(config.USABLE_SENSORS):
|
||||
rows = multi[multi["sensor"] == sensor]
|
||||
last = density[density["sensor"] == sensor]["time"].max()
|
||||
out.append(
|
||||
{
|
||||
"id": sensor,
|
||||
"name": config.DISPLAY_NAMES[sensor],
|
||||
"lat": round(float(rows["latitude"].median()), 6),
|
||||
"lon": round(float(rows["longitude"].median()), 6),
|
||||
"mean": round(float(rows["value"].mean()), 1),
|
||||
"p95": round(float(rows["value"].quantile(0.95)), 1),
|
||||
"battery": (
|
||||
int(round(float(battery_latest.loc[sensor, "value"])))
|
||||
if sensor in battery_latest.index
|
||||
else None
|
||||
),
|
||||
"last_seen": rfc3339(last),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def export_series(hourly: pd.DataFrame, data_version: str) -> dict:
|
||||
artifacts = config.artifacts_dir()
|
||||
series_dir = artifacts / "series"
|
||||
series_dir.mkdir(parents=True, exist_ok=True)
|
||||
per_sensor: dict[str, list[dict]] = {}
|
||||
for sensor in sorted(config.USABLE_SENSORS):
|
||||
frame = hourly[hourly["sensor"] == sensor]
|
||||
pts = _series_points(frame)
|
||||
per_sensor[sensor] = pts
|
||||
(series_dir / f"{sensor}.json").write_text(
|
||||
json.dumps(
|
||||
{"sensor": sensor, "step": "1h", "points": pts}, ensure_ascii=False
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
payload = {"step": "1h", "data_version": data_version, "series": per_sensor}
|
||||
(artifacts / "series.json").write_text(json.dumps(payload, ensure_ascii=False) + "\n")
|
||||
return payload
|
||||
|
||||
|
||||
def _weather_now(forecast: pd.DataFrame, meta: dict) -> dict:
|
||||
current = meta.get("current") or {}
|
||||
now_t = current.get("t")
|
||||
if now_t:
|
||||
ts = pd.Timestamp(now_t, tz="UTC") if pd.Timestamp(now_t).tz is None else pd.Timestamp(now_t).tz_convert("UTC")
|
||||
else:
|
||||
ts = pd.Timestamp(meta["fetched_at"]).tz_convert("UTC")
|
||||
row = forecast.iloc[(forecast["time"] - ts).abs().argmin()]
|
||||
return {
|
||||
"t": rfc3339(ts),
|
||||
"temp_c": round(float(current.get("temp_c", row["temp_c"])), 1),
|
||||
"precip_mm": round(float(current.get("precip_mm", row["precip_mm"])), 2),
|
||||
"cloud_pct": int(round(float(current.get("cloud_pct", row["cloud_pct"])))),
|
||||
"wind_kmh": round(float(current.get("wind_kmh", row["wind_kmh"])), 1),
|
||||
"humidity_pct": int(round(float(current.get("humidity_pct", row["humidity_pct"])))),
|
||||
}
|
||||
|
||||
|
||||
def export_context(forecast: pd.DataFrame, meta: dict) -> dict:
|
||||
now = _weather_now(forecast, meta)
|
||||
start = pd.Timestamp(now["t"]).floor("h")
|
||||
window = forecast[(forecast["time"] >= start) & (forecast["time"] < start + pd.Timedelta(days=7))]
|
||||
points = [
|
||||
{
|
||||
"t": rfc3339(row.time),
|
||||
"temp_c": round(float(row.temp_c), 1),
|
||||
"precip_mm": round(float(row.precip_mm), 2),
|
||||
"cloud_pct": int(round(float(row.cloud_pct))),
|
||||
"wind_kmh": round(float(row.wind_kmh), 1),
|
||||
"humidity_pct": int(round(float(row.humidity_pct))),
|
||||
}
|
||||
for row in window.itertuples(index=False)
|
||||
]
|
||||
holidays = [
|
||||
{"date": date, "name": name} for date, name in sorted(config.HOLIDAYS.items())
|
||||
]
|
||||
payload = {
|
||||
"weather": {"now": now, "forecast": points},
|
||||
"holidays": holidays,
|
||||
"updated_at": meta["fetched_at"],
|
||||
"cached": False,
|
||||
}
|
||||
(config.artifacts_dir() / "context.json").write_text(
|
||||
json.dumps(payload, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def run_data(*, refresh_weather: bool = False) -> dict:
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
artifacts = config.artifacts_dir()
|
||||
cache = config.cache_dir()
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
raw = load_raw()
|
||||
density, battery, audit = clean(raw)
|
||||
write_audit(audit)
|
||||
density[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
||||
cache / "density_clean.parquet", index=False
|
||||
)
|
||||
battery[["time", "sensor", "sensor_id", "value"]].to_parquet(
|
||||
cache / "battery_clean.parquet", index=False
|
||||
)
|
||||
log.info(
|
||||
"parsed %s rows; %s usable sensors; %s density rows",
|
||||
audit["raw_rows"],
|
||||
audit["usable_sensors"],
|
||||
len(density),
|
||||
)
|
||||
|
||||
hourly = to_hourly(density)
|
||||
hourly.to_parquet(cache / "hourly.parquet", index=False)
|
||||
|
||||
archive, forecast, meta = ensure_weather(
|
||||
density["time"].min(), density["time"].max(), refresh=refresh_weather
|
||||
)
|
||||
features = build_features(hourly, archive)
|
||||
features.to_parquet(cache / "hourly_features.parquet", index=False)
|
||||
log.info("built %s hourly feature rows", len(features))
|
||||
|
||||
data_version = minute_iso(audit["time_range"]["to"])
|
||||
sensors = export_sensors(density, battery)
|
||||
(artifacts / "sensors.json").write_text(
|
||||
json.dumps(sensors, indent=2, ensure_ascii=False) + "\n"
|
||||
)
|
||||
series = export_series(hourly, data_version)
|
||||
context = export_context(forecast, meta)
|
||||
|
||||
summary = {
|
||||
"sensors": len(sensors),
|
||||
"series_sensors": len(series["series"]),
|
||||
"hourly_rows": int(len(hourly)),
|
||||
"feature_rows": int(len(features)),
|
||||
"weather_rows": int(len(archive)),
|
||||
"context_forecast_hours": len(context["weather"]["forecast"]),
|
||||
"data_version": data_version,
|
||||
"audit_ok": audit["all_checks_ok"],
|
||||
}
|
||||
log.info("data artifacts written to %s: %s", artifacts, json.dumps(summary))
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--refresh-weather", action="store_true")
|
||||
args = parser.parse_args()
|
||||
run_data(refresh_weather=args.refresh_weather)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user