chore: bootstrap ML-py scaffold
This commit is contained in:
commit
2eeb1f937a
12 files changed
+1924
No files matched your search
@@ -0,0 +1,3 @@
|
||||
"""PulseCast ML package."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Configuration: paths, sensors, holidays, scenarios, contract versions.
|
||||
|
||||
All timestamps are UTC internally; Europe/Berlin is used only for calendar
|
||||
features and display-oriented payloads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# --------------------------------------------------------------------------- paths
|
||||
|
||||
|
||||
def _root() -> Path:
|
||||
"""Project root: PULSECAST_ROOT, else the first parent containing pyproject.toml.
|
||||
|
||||
Works both from a checkout (src/pulsecast/config.py -> repo root) and from
|
||||
``python -m`` / pytest launched at the repo root.
|
||||
"""
|
||||
env = os.environ.get("PULSECAST_ROOT")
|
||||
if env:
|
||||
return Path(env).resolve()
|
||||
here = Path(__file__).resolve()
|
||||
for parent in here.parents:
|
||||
if (parent / "pyproject.toml").exists():
|
||||
return parent
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def artifacts_dir() -> Path:
|
||||
return Path(os.environ.get("PULSECAST_ARTIFACTS", _root() / "artifacts")).resolve()
|
||||
|
||||
|
||||
def cache_dir() -> Path:
|
||||
return Path(os.environ.get("PULSECAST_CACHE", _root() / "data" / "cache")).resolve()
|
||||
|
||||
|
||||
def raw_csv_path() -> Path:
|
||||
return Path(
|
||||
os.environ.get(
|
||||
"PULSECAST_RAW_CSV",
|
||||
"/home/noel/Documents/Hackathon_2026/data/raw/schwarz-pax.csv",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- data
|
||||
|
||||
# Multi-sensor window documented in mission/data.md. density-01 has older
|
||||
# history but the other seven locations start on 2026-06-12.
|
||||
MULTI_SENSOR_START = "2026-06-12T00:00:00Z"
|
||||
|
||||
# 28-day time-based holdout, measured back from the last observed hour.
|
||||
TEST_DAYS = 28
|
||||
|
||||
# Excluded devices, with the reason recorded in the data audit.
|
||||
EXCLUDED_SENSORS: dict[str, str] = {
|
||||
"maker-fair": "test installation in Hannover, 73 readings, outside the inner city",
|
||||
"niklas-01": "8 readings, outside the inner-city cluster",
|
||||
"hn-bollwerksturm-01": "5 readings, insufficient coverage",
|
||||
}
|
||||
|
||||
# Two physical device IDs at one location; merge into one logical sensor.
|
||||
DEVICE_TO_SENSOR: dict[str, str] = {
|
||||
"863f75b0": "density-01",
|
||||
"f8c19730": "density-01",
|
||||
}
|
||||
|
||||
USABLE_SENSORS: tuple[str, ...] = (
|
||||
"density-01",
|
||||
"hn-goetzentuem-01",
|
||||
"hn-hagenbuchersee-01",
|
||||
"hn-hagenbuchersee-02",
|
||||
"hn-marahaus-01",
|
||||
"hn-neckarufer-01",
|
||||
"hn-soleo-01",
|
||||
"hn-stadtgalerie-01",
|
||||
)
|
||||
|
||||
DISPLAY_NAMES: dict[str, str] = {
|
||||
"density-01": "Density 01",
|
||||
"hn-goetzentuem-01": "Götzenturm",
|
||||
"hn-hagenbuchersee-01": "Hagenbuchersee 1",
|
||||
"hn-hagenbuchersee-02": "Hagenbuchersee 2",
|
||||
"hn-marahaus-01": "Marahaus",
|
||||
"hn-neckarufer-01": "Neckarufer",
|
||||
"hn-soleo-01": "Soleo",
|
||||
"hn-stadtgalerie-01": "Stadtgalerie",
|
||||
}
|
||||
|
||||
# Reject rows outside this plausible range before aggregation.
|
||||
VALUE_MIN = 0.0
|
||||
VALUE_MAX = 1000.0
|
||||
|
||||
# --------------------------------------------------------------------------- context
|
||||
|
||||
# Heilbronn inner city (Stadtgalerie / Kaiserstraße area). Open-Meteo snaps to
|
||||
# its nearest grid cell; document the returned coordinates in context.json.
|
||||
LATITUDE = 49.1427
|
||||
LONGITUDE = 9.2109
|
||||
TIMEZONE = "Europe/Berlin"
|
||||
|
||||
WEATHER_FIELDS = (
|
||||
"temperature_2m",
|
||||
"precipitation",
|
||||
"cloud_cover",
|
||||
"wind_speed_10m",
|
||||
"relative_humidity_2m",
|
||||
)
|
||||
WEATHER_COLUMNS = {
|
||||
"temperature_2m": "temp_c",
|
||||
"precipitation": "precip_mm",
|
||||
"cloud_cover": "cloud_pct",
|
||||
"wind_speed_10m": "wind_kmh",
|
||||
"relative_humidity_2m": "humidity_pct",
|
||||
}
|
||||
|
||||
# Baden-Württemberg public holidays, 2025-2026 (mission/data.md). Kept explicit
|
||||
# so the build is deterministic and needs no holiday library.
|
||||
HOLIDAYS: dict[str, str] = {
|
||||
"2025-01-01": "Neujahr",
|
||||
"2025-01-06": "Heilige Drei Könige",
|
||||
"2025-04-18": "Karfreitag",
|
||||
"2025-04-21": "Ostermontag",
|
||||
"2025-05-01": "Tag der Arbeit",
|
||||
"2025-05-29": "Christi Himmelfahrt",
|
||||
"2025-06-09": "Pfingstmontag",
|
||||
"2025-06-19": "Fronleichnam",
|
||||
"2025-10-03": "Tag der Deutschen Einheit",
|
||||
"2025-11-01": "Allerheiligen",
|
||||
"2025-12-25": "1. Weihnachtstag",
|
||||
"2025-12-26": "2. Weihnachtstag",
|
||||
"2026-01-01": "Neujahr",
|
||||
"2026-01-06": "Heilige Drei Könige",
|
||||
"2026-04-03": "Karfreitag",
|
||||
"2026-04-06": "Ostermontag",
|
||||
"2026-05-01": "Tag der Arbeit",
|
||||
"2026-05-14": "Christi Himmelfahrt",
|
||||
"2026-05-25": "Pfingstmontag",
|
||||
"2026-06-04": "Fronleichnam",
|
||||
"2026-10-03": "Tag der Deutschen Einheit",
|
||||
"2026-11-01": "Allerheiligen",
|
||||
"2026-12-25": "1. Weihnachtstag",
|
||||
"2026-12-26": "2. Weihnachtstag",
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------------------- model
|
||||
|
||||
FEATURE_CALENDAR = ["hour", "weekday", "month", "is_weekend", "is_holiday", "sensor_code"]
|
||||
FEATURE_CALENDAR_NO_HOLIDAY = ["hour", "weekday", "month", "is_weekend", "sensor_code"]
|
||||
|
||||
# Day-ahead: every feature is at least 24 h old.
|
||||
FEATURE_DAY = FEATURE_CALENDAR + [
|
||||
"lag_day",
|
||||
"lag_2d",
|
||||
"lag_week",
|
||||
"seasonal_mean_4w",
|
||||
"roll_24",
|
||||
"roll_168",
|
||||
]
|
||||
# 7-day-ahead: every feature is at least 7 days old, so a 168 h forecast is
|
||||
# possible without recursion.
|
||||
FEATURE_WEEK = FEATURE_CALENDAR + ["lag_week", "seasonal_mean_4w"]
|
||||
FEATURE_WEATHER = ["temp_c", "precip_mm"]
|
||||
|
||||
FEATURE_DAY_NO_HOLIDAY = FEATURE_CALENDAR_NO_HOLIDAY + FEATURE_DAY[6:]
|
||||
FEATURE_DAY_NO_WEATHER = FEATURE_DAY
|
||||
FEATURE_DAY_WEATHER = FEATURE_DAY + FEATURE_WEATHER
|
||||
FEATURE_DAY_HOLIDAY = FEATURE_DAY + ["is_holiday"] # is_holiday already present; kept for clarity
|
||||
FEATURE_WEEK_NO_WEATHER = FEATURE_WEEK
|
||||
FEATURE_WEEK_WEATHER = FEATURE_WEEK + FEATURE_WEATHER
|
||||
|
||||
# Scenario presets. Weather adjustments apply on the event's weekdays; the
|
||||
# event uplift is a transparent multiplicative overlay applied after the model.
|
||||
# These are decision-support assumptions, not causal estimates.
|
||||
SCENARIOS: dict[str, dict] = {
|
||||
"base": {
|
||||
"label": "Base forecast",
|
||||
"temp_delta_c": 0.0,
|
||||
"precip_mm": None,
|
||||
"event": None,
|
||||
},
|
||||
"sunny-saturday-market": {
|
||||
"label": "Sunny Saturday + Christmas market",
|
||||
"temp_delta_c": 4.0,
|
||||
"precip_mm": 0.0,
|
||||
"event": {"type": "market", "uplift": 0.15, "weekdays": [5], "hours": list(range(10, 21))},
|
||||
},
|
||||
"rainy-saturday": {
|
||||
"label": "Rainy Saturday",
|
||||
"temp_delta_c": -3.0,
|
||||
"precip_mm": 4.0,
|
||||
"event": {"type": "rain", "uplift": -0.15, "weekdays": [5], "hours": list(range(10, 21))},
|
||||
},
|
||||
"weekday-event": {
|
||||
"label": "Weekday evening event",
|
||||
"temp_delta_c": 0.0,
|
||||
"precip_mm": None,
|
||||
"event": {"type": "event", "uplift": 0.10, "weekdays": [0, 1, 2, 3, 4], "hours": [17, 18, 19, 20]},
|
||||
},
|
||||
}
|
||||
|
||||
HORIZONS = ("15m", "1h", "48h", "7d")
|
||||
HORIZON_STEPS = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
|
||||
|
||||
LIMITS = [
|
||||
"estimated devices, not people",
|
||||
"8 sensors in a ~580 m inner-city cluster",
|
||||
"5 months of multi-sensor data; no annual seasonality claim",
|
||||
"observed weather in training, forecast weather in scenarios",
|
||||
"holidays are context, not a large accuracy claim",
|
||||
]
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Hourly aggregation and leakage-safe feature construction.
|
||||
|
||||
Design rules:
|
||||
- Hour bins are UTC hours (unambiguous across DST); local calendar fields are
|
||||
derived from Europe/Berlin.
|
||||
- Every lag feature is at least 24 h old (day-ahead) or 168 h old (7-day).
|
||||
Nothing at hour t uses the target at hour t or later.
|
||||
- Seasonal lags use the previous *observed* same weekday/hour, so sensor gaps
|
||||
do not silently break the feature.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from . import config
|
||||
|
||||
SENSOR_CODES = {name: i for i, name in enumerate(config.USABLE_SENSORS)}
|
||||
|
||||
|
||||
def to_hourly(density: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Aggregate 15-minute density readings to UTC hourly means per sensor."""
|
||||
df = density.copy()
|
||||
df["t"] = df["time"].dt.floor("h")
|
||||
hourly = (
|
||||
df.groupby(["sensor", "t"], observed=True)
|
||||
.agg(v=("value", "mean"), n=("value", "size"))
|
||||
.reset_index()
|
||||
)
|
||||
hourly["v"] = hourly["v"].astype(float)
|
||||
return hourly.sort_values(["sensor", "t"]).reset_index(drop=True)
|
||||
|
||||
|
||||
def _calendar(frame: pd.DataFrame, time_col: str = "t") -> pd.DataFrame:
|
||||
local = frame[time_col].dt.tz_convert(config.TIMEZONE)
|
||||
out = frame.copy()
|
||||
out["local_date"] = local.dt.strftime("%Y-%m-%d")
|
||||
out["hour"] = local.dt.hour.astype("int8")
|
||||
out["weekday"] = local.dt.day_of_week.astype("int8")
|
||||
out["month"] = local.dt.month.astype("int8")
|
||||
out["is_weekend"] = (out["weekday"] >= 5).astype("int8")
|
||||
out["holiday_name"] = out["local_date"].map(config.HOLIDAYS)
|
||||
out["is_holiday"] = out["holiday_name"].notna().astype("int8")
|
||||
return out
|
||||
|
||||
|
||||
def _grid(hourly: pd.DataFrame, end: pd.Timestamp) -> pd.DataFrame:
|
||||
"""Continuous hourly grid per sensor; used only for time-based rollups."""
|
||||
parts = []
|
||||
for sensor, group in hourly.groupby("sensor", observed=True):
|
||||
idx = pd.date_range(group["t"].min(), end, freq="h", tz="UTC")
|
||||
s = group.set_index("t")["v"].reindex(idx)
|
||||
part = pd.DataFrame({"sensor": sensor, "t": idx, "y": s.to_numpy()})
|
||||
parts.append(part)
|
||||
grid = pd.concat(parts, ignore_index=True)
|
||||
grid = grid.sort_values(["sensor", "t"]).reset_index(drop=True)
|
||||
return grid
|
||||
|
||||
|
||||
def _add_rollups(grid: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Rolling means shifted 24 h back: only yesterday-and-older information."""
|
||||
out = []
|
||||
for _, group in grid.groupby("sensor", observed=True):
|
||||
g = group.sort_values("t").copy()
|
||||
shifted = g["y"].shift(24)
|
||||
g["roll_24"] = shifted.rolling(24, min_periods=12).mean()
|
||||
g["roll_168"] = shifted.rolling(168, min_periods=84).mean()
|
||||
out.append(g)
|
||||
return pd.concat(out, ignore_index=True)
|
||||
|
||||
|
||||
def _add_seasonal_lags(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Previous observed same weekday/hour values, strictly in the past."""
|
||||
out = df.sort_values(["sensor", "weekday", "hour", "t"]).copy()
|
||||
grouped = out.groupby(["sensor", "weekday", "hour"], observed=True)["y"]
|
||||
out["lag_week"] = grouped.shift(1)
|
||||
prior = pd.concat(
|
||||
[grouped.shift(k) for k in (2, 3, 4)],
|
||||
axis=1,
|
||||
)
|
||||
prior.columns = ["lag_2w", "lag_3w", "lag_4w"]
|
||||
all_prior = pd.concat([out["lag_week"], prior], axis=1)
|
||||
counts = all_prior.notna().sum(axis=1)
|
||||
out["seasonal_mean_4w"] = all_prior.mean(axis=1, skipna=True).where(counts >= 2)
|
||||
|
||||
by_hour = out.groupby(["sensor", "hour"], observed=True)["y"]
|
||||
out["lag_day"] = by_hour.shift(1)
|
||||
out["lag_2d"] = by_hour.shift(2)
|
||||
return out.sort_values(["sensor", "t"]).reset_index(drop=True)
|
||||
|
||||
|
||||
def build_features(
|
||||
hourly: pd.DataFrame,
|
||||
weather: pd.DataFrame,
|
||||
*,
|
||||
data_end: pd.Timestamp | None = None,
|
||||
) -> pd.DataFrame:
|
||||
"""Build the full feature table (observed hours only, target present)."""
|
||||
hourly = hourly[hourly["v"].notna()].copy()
|
||||
end = data_end or hourly["t"].max()
|
||||
grid = _add_rollups(_grid(hourly, end))
|
||||
df = hourly.merge(
|
||||
grid[["sensor", "t", "y", "roll_24", "roll_168"]],
|
||||
on=["sensor", "t"],
|
||||
how="left",
|
||||
)
|
||||
df = _calendar(df)
|
||||
df["sensor_code"] = df["sensor"].map(SENSOR_CODES).astype("int16")
|
||||
df = _add_seasonal_lags(df)
|
||||
df = df.merge(weather.drop(columns=[c for c in ("time",) if c in weather.columns]).assign(
|
||||
t=weather["time"]
|
||||
), on="t", how="left")
|
||||
df = df[df["y"].notna()].copy()
|
||||
return df.sort_values(["sensor", "t"]).reset_index(drop=True)
|
||||
|
||||
|
||||
def feature_columns(kind: str) -> list[str]:
|
||||
mapping = {
|
||||
"day_no_holiday": config.FEATURE_DAY_NO_HOLIDAY,
|
||||
"day": config.FEATURE_DAY,
|
||||
"day_weather": config.FEATURE_DAY_WEATHER,
|
||||
"week": config.FEATURE_WEEK,
|
||||
"week_weather": config.FEATURE_WEEK_WEATHER,
|
||||
}
|
||||
try:
|
||||
return list(mapping[kind])
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"unknown feature set {kind!r}") from exc
|
||||
|
||||
|
||||
def seasonal_lookup(hourly: pd.DataFrame) -> dict[str, dict[str, float]]:
|
||||
"""Last-4-observation lookup per sensor/weekday/hour for future inference.
|
||||
|
||||
Key format: "<sensor>|<weekday>|<hour>" -> {"mean4": float, "last": float}.
|
||||
This is the same definition used by ``lag_week`` / ``seasonal_mean_4w`` in
|
||||
training, evaluated at the last observed week, so a future timestamp can use
|
||||
it without assuming data after the CSV end.
|
||||
"""
|
||||
df = _calendar(hourly[hourly["v"].notna()].copy())
|
||||
lookup: dict[str, dict[str, float]] = {}
|
||||
for (sensor, weekday, hour), group in df.groupby(
|
||||
["sensor", "weekday", "hour"], observed=True
|
||||
):
|
||||
last4 = group.sort_values("t")["v"].tail(4)
|
||||
lookup[f"{sensor}|{int(weekday)}|{int(hour)}"] = {
|
||||
"mean4": round(float(last4.mean()), 4),
|
||||
"last": round(float(last4.iloc[-1]), 4),
|
||||
}
|
||||
return lookup
|
||||
|
||||
|
||||
def future_calendar(start: pd.Timestamp, periods: int) -> pd.DataFrame:
|
||||
"""Calendar features for a future hourly grid (no target, no lags)."""
|
||||
t = pd.date_range(start, periods=periods, freq="h", tz="UTC")
|
||||
frame = pd.DataFrame({"t": t})
|
||||
frame = _calendar(frame)
|
||||
return frame
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Parse and clean the raw PAX CSV.
|
||||
|
||||
Verifies the claims in mission/data.md: 192,464 rows, 12 device IDs, 8 usable
|
||||
logical locations, three excluded test devices, and the density-01 merge.
|
||||
The raw file is never written to git; a cached parquet copy lives in data/cache/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from . import config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
EXPECTED_CHECKS = {
|
||||
"raw_rows": 192_464,
|
||||
"density_rows": 96_232,
|
||||
"battery_rows": 96_232,
|
||||
"device_ids": 12,
|
||||
"sensor_names": 11,
|
||||
"usable_sensors": 8,
|
||||
"density_01_rows_multi_sensor": 12_670,
|
||||
}
|
||||
|
||||
|
||||
def _logical_sensor(name: str, device_id: str) -> str:
|
||||
return config.DEVICE_TO_SENSOR.get(device_id, name)
|
||||
|
||||
|
||||
def load_raw(path: Path | None = None) -> pd.DataFrame:
|
||||
path = path or config.raw_csv_path()
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"raw CSV not found at {path}. Set PULSECAST_RAW_CSV or place the "
|
||||
"challenge download at that path (never commit it)."
|
||||
)
|
||||
df = pd.read_csv(path, parse_dates=["time"], dtype={"sensor_id": "string", "name": "string"})
|
||||
df["time"] = pd.to_datetime(df["time"], utc=True)
|
||||
expected = {"time", "sensor_id", "name", "longitude", "latitude", "type", "value"}
|
||||
missing = expected - set(df.columns)
|
||||
if missing:
|
||||
raise ValueError(f"raw CSV missing columns: {sorted(missing)}")
|
||||
return df
|
||||
|
||||
|
||||
def clean(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
||||
"""Return (density, battery, audit) with logical sensors and exclusions applied."""
|
||||
density = df[df["type"] == "densityData"].copy()
|
||||
battery = df[df["type"] == "batteryData"].copy()
|
||||
|
||||
for frame in (density, battery):
|
||||
frame["sensor"] = [
|
||||
_logical_sensor(n, d) for n, d in zip(frame["name"], frame["sensor_id"], strict=True)
|
||||
]
|
||||
|
||||
excluded_rows = {
|
||||
name: int((density["sensor"] == name).sum())
|
||||
for name in config.EXCLUDED_SENSORS
|
||||
}
|
||||
density = density[~density["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
|
||||
battery = battery[~battery["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
|
||||
|
||||
before = len(density)
|
||||
density = density[
|
||||
density["value"].between(config.VALUE_MIN, config.VALUE_MAX, inclusive="both")
|
||||
].copy()
|
||||
dropped_out_of_range = before - len(density)
|
||||
|
||||
usable = sorted(density["sensor"].unique().tolist())
|
||||
if set(usable) != set(config.USABLE_SENSORS):
|
||||
raise ValueError(
|
||||
f"usable sensor set changed: {usable}; expected {sorted(config.USABLE_SENSORS)}"
|
||||
)
|
||||
|
||||
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
|
||||
density_01_devices = (
|
||||
multi[multi["sensor"] == "density-01"]
|
||||
.groupby("sensor_id", observed=True)
|
||||
.agg(rows=("value", "size"), first=("time", "min"), last=("time", "max"))
|
||||
)
|
||||
|
||||
# One reading per (time, device, type); duplicates would double-weight a bin.
|
||||
dup_keys = int(density.duplicated(subset=["time", "sensor_id"]).sum())
|
||||
if dup_keys:
|
||||
density = density.drop_duplicates(subset=["time", "sensor_id"], keep="last")
|
||||
|
||||
cadence = (
|
||||
density.sort_values(["sensor", "time"])
|
||||
.groupby("sensor", observed=True)["time"]
|
||||
.diff()
|
||||
.dt.total_seconds()
|
||||
.div(60)
|
||||
.dropna()
|
||||
)
|
||||
|
||||
audit = {
|
||||
"raw_rows": int(len(df)),
|
||||
"density_rows": int(len(df[df["type"] == "densityData"])),
|
||||
"battery_rows": int(len(df[df["type"] == "batteryData"])),
|
||||
"device_ids": int(df["sensor_id"].nunique()),
|
||||
"sensor_names": int(df["name"].nunique()),
|
||||
"usable_sensors": len(usable),
|
||||
"sensor_ids": usable,
|
||||
"excluded_sensors": {
|
||||
name: {"rows": excluded_rows[name], "reason": reason}
|
||||
for name, reason in config.EXCLUDED_SENSORS.items()
|
||||
},
|
||||
"density_01_devices": {
|
||||
str(device): {
|
||||
"rows": int(row["rows"]),
|
||||
"first": _iso(row["first"]),
|
||||
"last": _iso(row["last"]),
|
||||
}
|
||||
for device, row in density_01_devices.iterrows()
|
||||
},
|
||||
"density_01_rows_multi_sensor": int(
|
||||
(multi["sensor"] == "density-01").sum()
|
||||
),
|
||||
"time_range": {
|
||||
"from": _iso(density["time"].min()),
|
||||
"to": _iso(density["time"].max()),
|
||||
},
|
||||
"multi_sensor_from": _iso(multi["time"].min()),
|
||||
"cadence_median_min": round(float(cadence.median()), 2),
|
||||
"dropped_out_of_range": int(dropped_out_of_range),
|
||||
"duplicate_keys_dropped": dup_keys,
|
||||
"checks": [],
|
||||
"all_checks_ok": True,
|
||||
}
|
||||
for key, expected in EXPECTED_CHECKS.items():
|
||||
actual = audit[key]
|
||||
ok = int(actual) == int(expected)
|
||||
audit["checks"].append({"name": key, "expected": expected, "actual": actual, "ok": ok})
|
||||
if not ok:
|
||||
audit["all_checks_ok"] = False
|
||||
log.warning("data audit mismatch %s: expected %s, got %s", key, expected, actual)
|
||||
return density, battery, audit
|
||||
|
||||
|
||||
def write_audit(audit: dict, path: Path | None = None) -> Path:
|
||||
path = path or config.artifacts_dir() / "data_audit.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(audit, indent=2, ensure_ascii=False) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def load_cached_density() -> pd.DataFrame:
|
||||
path = config.cache_dir() / "density_clean.parquet"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"{path} missing; run `make data` first")
|
||||
return pd.read_parquet(path)
|
||||
|
||||
|
||||
def _iso(ts) -> str | None:
|
||||
if ts is None or pd.isna(ts):
|
||||
return None
|
||||
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Open-Meteo weather: historical archive for training, forecast for scenarios.
|
||||
|
||||
No API key. Responses are cached under data/cache/ so `make train` and the
|
||||
Docker image need no network. The raw CSV and cache stay out of git; the final
|
||||
context.json is committed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pandas as pd
|
||||
|
||||
from . import config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive"
|
||||
FORECAST_URL = "https://api.open-meteo.com/v1/forecast"
|
||||
ARCHIVE_FILE = "weather_archive.parquet"
|
||||
FORECAST_FILE = "weather_forecast.parquet"
|
||||
META_FILE = "weather_meta.json"
|
||||
HOURLY_PARAMS = ",".join(config.WEATHER_FIELDS)
|
||||
CURRENT_PARAMS = HOURLY_PARAMS
|
||||
|
||||
|
||||
def _get_json(url: str, params: dict, *, retries: int = 3) -> dict:
|
||||
last: Exception | None = None
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
resp = client.get(url, params=params)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
if payload.get("error"):
|
||||
raise RuntimeError(f"Open-Meteo error: {payload.get('reason', payload)}")
|
||||
return payload
|
||||
except Exception as exc: # network / HTTP / payload
|
||||
last = exc
|
||||
if attempt < retries - 1:
|
||||
time.sleep(2**attempt)
|
||||
raise RuntimeError(f"Open-Meteo request failed after {retries} attempts: {last}")
|
||||
|
||||
|
||||
def _hourly_frame(payload: dict) -> pd.DataFrame:
|
||||
hourly = payload.get("hourly") or {}
|
||||
if "time" not in hourly:
|
||||
raise RuntimeError("Open-Meteo payload has no hourly.time")
|
||||
frame = pd.DataFrame(hourly)
|
||||
frame["time"] = pd.to_datetime(frame["time"], utc=True)
|
||||
missing = [f for f in config.WEATHER_FIELDS if f not in frame.columns]
|
||||
if missing:
|
||||
raise RuntimeError(f"Open-Meteo payload missing fields: {missing}")
|
||||
frame = frame.rename(columns=config.WEATHER_COLUMNS)
|
||||
cols = ["time"] + list(config.WEATHER_COLUMNS.values())
|
||||
return frame[cols]
|
||||
|
||||
|
||||
def _meta_path() -> Path:
|
||||
return config.cache_dir() / META_FILE
|
||||
|
||||
|
||||
def _load_meta() -> dict:
|
||||
path = _meta_path()
|
||||
if path.exists():
|
||||
return json.loads(path.read_text())
|
||||
return {}
|
||||
|
||||
|
||||
def fetch_archive(start: date, end: date) -> pd.DataFrame:
|
||||
payload = _get_json(
|
||||
ARCHIVE_URL,
|
||||
{
|
||||
"latitude": config.LATITUDE,
|
||||
"longitude": config.LONGITUDE,
|
||||
"start_date": start.isoformat(),
|
||||
"end_date": end.isoformat(),
|
||||
"hourly": HOURLY_PARAMS,
|
||||
"timezone": "UTC",
|
||||
},
|
||||
)
|
||||
frame = _hourly_frame(payload)
|
||||
log.info("fetched %s archive weather rows for %s..%s", len(frame), start, end)
|
||||
return frame
|
||||
|
||||
|
||||
def fetch_forecast(days: int = 8) -> tuple[pd.DataFrame, dict, dict]:
|
||||
payload = _get_json(
|
||||
FORECAST_URL,
|
||||
{
|
||||
"latitude": config.LATITUDE,
|
||||
"longitude": config.LONGITUDE,
|
||||
"hourly": HOURLY_PARAMS,
|
||||
"current": CURRENT_PARAMS,
|
||||
"forecast_days": days,
|
||||
"timezone": "UTC",
|
||||
},
|
||||
)
|
||||
frame = _hourly_frame(payload)
|
||||
current = payload.get("current") or {}
|
||||
now = {
|
||||
"t": current.get("time"),
|
||||
"temp_c": current.get("temperature_2m"),
|
||||
"precip_mm": current.get("precipitation"),
|
||||
"cloud_pct": current.get("cloud_cover"),
|
||||
"wind_kmh": current.get("wind_speed_10m"),
|
||||
"humidity_pct": current.get("relative_humidity_2m"),
|
||||
}
|
||||
grid = {"latitude": payload.get("latitude"), "longitude": payload.get("longitude")}
|
||||
meta = {
|
||||
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"current": now,
|
||||
"grid": grid,
|
||||
"source": "open-meteo",
|
||||
}
|
||||
log.info("fetched %s forecast weather rows", len(frame))
|
||||
return frame, meta, grid
|
||||
|
||||
|
||||
def ensure_weather(
|
||||
data_start: pd.Timestamp | None = None,
|
||||
data_end: pd.Timestamp | None = None,
|
||||
*,
|
||||
refresh: bool = False,
|
||||
) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
||||
"""Load cached weather, fetching only when missing, stale, or refresh=True."""
|
||||
cache = config.cache_dir()
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
archive_path = cache / ARCHIVE_FILE
|
||||
forecast_path = cache / FORECAST_FILE
|
||||
meta = _load_meta()
|
||||
|
||||
if archive_path.exists() and not refresh:
|
||||
archive = pd.read_parquet(archive_path)
|
||||
else:
|
||||
start = (data_start or pd.Timestamp(config.MULTI_SENSOR_START)).date() - timedelta(days=2)
|
||||
end = (data_end or pd.Timestamp.now(tz="UTC")).date() + timedelta(days=1)
|
||||
archive = fetch_archive(start, end)
|
||||
archive.to_parquet(archive_path, index=False)
|
||||
|
||||
if forecast_path.exists() and not refresh:
|
||||
forecast = pd.read_parquet(forecast_path)
|
||||
else:
|
||||
try:
|
||||
forecast, meta, _ = fetch_forecast()
|
||||
forecast.to_parquet(forecast_path, index=False)
|
||||
_meta_path().write_text(json.dumps(meta, indent=2) + "\n")
|
||||
except Exception:
|
||||
if not forecast_path.exists():
|
||||
raise
|
||||
log.warning("Open-Meteo forecast refresh failed; using cached forecast")
|
||||
forecast = pd.read_parquet(forecast_path)
|
||||
|
||||
if not meta:
|
||||
meta = {
|
||||
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"current": None,
|
||||
"grid": {"latitude": config.LATITUDE, "longitude": config.LONGITUDE},
|
||||
"source": "cache",
|
||||
}
|
||||
|
||||
# Keep the archive within the challenge window (+/- 2 days) to avoid
|
||||
# accidentally training on future weather if the cache is refreshed later.
|
||||
if data_end is not None:
|
||||
archive = archive[archive["time"] <= data_end + pd.Timedelta(days=1)].copy()
|
||||
if data_start is not None:
|
||||
archive = archive[archive["time"] >= data_start - pd.Timedelta(days=2)].copy()
|
||||
return archive.reset_index(drop=True), forecast.reset_index(drop=True), meta
|
||||
|
||||
|
||||
def load_cached_weather() -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
||||
cache = config.cache_dir()
|
||||
archive = pd.read_parquet(cache / ARCHIVE_FILE)
|
||||
forecast = pd.read_parquet(cache / FORECAST_FILE)
|
||||
return archive, forecast, _load_meta()
|
||||
|
||||
|
||||
def refresh_weather() -> None:
|
||||
archive, forecast, meta = ensure_weather(refresh=True)
|
||||
log.info("weather refreshed: archive=%s forecast=%s", len(archive), len(forecast))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
refresh_weather()
|
||||
Reference in new issue
Block a user