chore: bootstrap ML-py scaffold
This commit is contained in:
commit
2eeb1f937a
12 files changed
+1924
No files matched your search
+23
@@ -0,0 +1,23 @@
|
|||||||
|
# Raw challenge data must never be committed.
|
||||||
|
data/raw/
|
||||||
|
data/cache/
|
||||||
|
|
||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
.venv/
|
||||||
|
.pytest_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
*.egg-info/
|
||||||
|
dist/
|
||||||
|
build/
|
||||||
|
|
||||||
|
# OS / editor
|
||||||
|
.DS_Store
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
# Local secrets / kubeconfigs
|
||||||
|
*.kubeconfig
|
||||||
|
kubeconfig*.yaml
|
||||||
|
.env
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
3.12
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
def main():
|
||||||
|
print("Hello from ml-py!")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://opencode.ai/config.json",
|
||||||
|
"model": "opencode-go/deepseek-v4.1-flash"
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
[project]
|
||||||
|
name = "pulsecast-ml"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "PulseCast ML: footfall pipeline, weather-aware forecast, scenario engine, FastAPI service"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.12"
|
||||||
|
dependencies = [
|
||||||
|
"fastapi>=0.162.7",
|
||||||
|
"httpx>=0.28.1",
|
||||||
|
"joblib>=1.6.0",
|
||||||
|
"numpy>=2.5.4",
|
||||||
|
"pandas>=3.1.1",
|
||||||
|
"pyarrow>=26.0.1",
|
||||||
|
"pytest>=9.2.0",
|
||||||
|
"scikit-learn>=1.9.1",
|
||||||
|
"uvicorn[standard]>=0.54.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["hatchling"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["src/pulsecast"]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
addopts = "-q"
|
||||||
|
|
||||||
|
[dependency-groups]
|
||||||
|
dev = []
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
"""PulseCast ML package."""
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
"""Configuration: paths, sensors, holidays, scenarios, contract versions.
|
||||||
|
|
||||||
|
All timestamps are UTC internally; Europe/Berlin is used only for calendar
|
||||||
|
features and display-oriented payloads.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- paths
|
||||||
|
|
||||||
|
|
||||||
|
def _root() -> Path:
|
||||||
|
"""Project root: PULSECAST_ROOT, else the first parent containing pyproject.toml.
|
||||||
|
|
||||||
|
Works both from a checkout (src/pulsecast/config.py -> repo root) and from
|
||||||
|
``python -m`` / pytest launched at the repo root.
|
||||||
|
"""
|
||||||
|
env = os.environ.get("PULSECAST_ROOT")
|
||||||
|
if env:
|
||||||
|
return Path(env).resolve()
|
||||||
|
here = Path(__file__).resolve()
|
||||||
|
for parent in here.parents:
|
||||||
|
if (parent / "pyproject.toml").exists():
|
||||||
|
return parent
|
||||||
|
return Path.cwd()
|
||||||
|
|
||||||
|
|
||||||
|
def artifacts_dir() -> Path:
|
||||||
|
return Path(os.environ.get("PULSECAST_ARTIFACTS", _root() / "artifacts")).resolve()
|
||||||
|
|
||||||
|
|
||||||
|
def cache_dir() -> Path:
|
||||||
|
return Path(os.environ.get("PULSECAST_CACHE", _root() / "data" / "cache")).resolve()
|
||||||
|
|
||||||
|
|
||||||
|
def raw_csv_path() -> Path:
|
||||||
|
return Path(
|
||||||
|
os.environ.get(
|
||||||
|
"PULSECAST_RAW_CSV",
|
||||||
|
"/home/noel/Documents/Hackathon_2026/data/raw/schwarz-pax.csv",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- data
|
||||||
|
|
||||||
|
# Multi-sensor window documented in mission/data.md. density-01 has older
|
||||||
|
# history but the other seven locations start on 2026-06-12.
|
||||||
|
MULTI_SENSOR_START = "2026-06-12T00:00:00Z"
|
||||||
|
|
||||||
|
# 28-day time-based holdout, measured back from the last observed hour.
|
||||||
|
TEST_DAYS = 28
|
||||||
|
|
||||||
|
# Excluded devices, with the reason recorded in the data audit.
|
||||||
|
EXCLUDED_SENSORS: dict[str, str] = {
|
||||||
|
"maker-fair": "test installation in Hannover, 73 readings, outside the inner city",
|
||||||
|
"niklas-01": "8 readings, outside the inner-city cluster",
|
||||||
|
"hn-bollwerksturm-01": "5 readings, insufficient coverage",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Two physical device IDs at one location; merge into one logical sensor.
|
||||||
|
DEVICE_TO_SENSOR: dict[str, str] = {
|
||||||
|
"863f75b0": "density-01",
|
||||||
|
"f8c19730": "density-01",
|
||||||
|
}
|
||||||
|
|
||||||
|
USABLE_SENSORS: tuple[str, ...] = (
|
||||||
|
"density-01",
|
||||||
|
"hn-goetzentuem-01",
|
||||||
|
"hn-hagenbuchersee-01",
|
||||||
|
"hn-hagenbuchersee-02",
|
||||||
|
"hn-marahaus-01",
|
||||||
|
"hn-neckarufer-01",
|
||||||
|
"hn-soleo-01",
|
||||||
|
"hn-stadtgalerie-01",
|
||||||
|
)
|
||||||
|
|
||||||
|
DISPLAY_NAMES: dict[str, str] = {
|
||||||
|
"density-01": "Density 01",
|
||||||
|
"hn-goetzentuem-01": "Götzenturm",
|
||||||
|
"hn-hagenbuchersee-01": "Hagenbuchersee 1",
|
||||||
|
"hn-hagenbuchersee-02": "Hagenbuchersee 2",
|
||||||
|
"hn-marahaus-01": "Marahaus",
|
||||||
|
"hn-neckarufer-01": "Neckarufer",
|
||||||
|
"hn-soleo-01": "Soleo",
|
||||||
|
"hn-stadtgalerie-01": "Stadtgalerie",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Reject rows outside this plausible range before aggregation.
|
||||||
|
VALUE_MIN = 0.0
|
||||||
|
VALUE_MAX = 1000.0
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- context
|
||||||
|
|
||||||
|
# Heilbronn inner city (Stadtgalerie / Kaiserstraße area). Open-Meteo snaps to
|
||||||
|
# its nearest grid cell; document the returned coordinates in context.json.
|
||||||
|
LATITUDE = 49.1427
|
||||||
|
LONGITUDE = 9.2109
|
||||||
|
TIMEZONE = "Europe/Berlin"
|
||||||
|
|
||||||
|
WEATHER_FIELDS = (
|
||||||
|
"temperature_2m",
|
||||||
|
"precipitation",
|
||||||
|
"cloud_cover",
|
||||||
|
"wind_speed_10m",
|
||||||
|
"relative_humidity_2m",
|
||||||
|
)
|
||||||
|
WEATHER_COLUMNS = {
|
||||||
|
"temperature_2m": "temp_c",
|
||||||
|
"precipitation": "precip_mm",
|
||||||
|
"cloud_cover": "cloud_pct",
|
||||||
|
"wind_speed_10m": "wind_kmh",
|
||||||
|
"relative_humidity_2m": "humidity_pct",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Baden-Württemberg public holidays, 2025-2026 (mission/data.md). Kept explicit
|
||||||
|
# so the build is deterministic and needs no holiday library.
|
||||||
|
HOLIDAYS: dict[str, str] = {
|
||||||
|
"2025-01-01": "Neujahr",
|
||||||
|
"2025-01-06": "Heilige Drei Könige",
|
||||||
|
"2025-04-18": "Karfreitag",
|
||||||
|
"2025-04-21": "Ostermontag",
|
||||||
|
"2025-05-01": "Tag der Arbeit",
|
||||||
|
"2025-05-29": "Christi Himmelfahrt",
|
||||||
|
"2025-06-09": "Pfingstmontag",
|
||||||
|
"2025-06-19": "Fronleichnam",
|
||||||
|
"2025-10-03": "Tag der Deutschen Einheit",
|
||||||
|
"2025-11-01": "Allerheiligen",
|
||||||
|
"2025-12-25": "1. Weihnachtstag",
|
||||||
|
"2025-12-26": "2. Weihnachtstag",
|
||||||
|
"2026-01-01": "Neujahr",
|
||||||
|
"2026-01-06": "Heilige Drei Könige",
|
||||||
|
"2026-04-03": "Karfreitag",
|
||||||
|
"2026-04-06": "Ostermontag",
|
||||||
|
"2026-05-01": "Tag der Arbeit",
|
||||||
|
"2026-05-14": "Christi Himmelfahrt",
|
||||||
|
"2026-05-25": "Pfingstmontag",
|
||||||
|
"2026-06-04": "Fronleichnam",
|
||||||
|
"2026-10-03": "Tag der Deutschen Einheit",
|
||||||
|
"2026-11-01": "Allerheiligen",
|
||||||
|
"2026-12-25": "1. Weihnachtstag",
|
||||||
|
"2026-12-26": "2. Weihnachtstag",
|
||||||
|
}
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- model
|
||||||
|
|
||||||
|
FEATURE_CALENDAR = ["hour", "weekday", "month", "is_weekend", "is_holiday", "sensor_code"]
|
||||||
|
FEATURE_CALENDAR_NO_HOLIDAY = ["hour", "weekday", "month", "is_weekend", "sensor_code"]
|
||||||
|
|
||||||
|
# Day-ahead: every feature is at least 24 h old.
|
||||||
|
FEATURE_DAY = FEATURE_CALENDAR + [
|
||||||
|
"lag_day",
|
||||||
|
"lag_2d",
|
||||||
|
"lag_week",
|
||||||
|
"seasonal_mean_4w",
|
||||||
|
"roll_24",
|
||||||
|
"roll_168",
|
||||||
|
]
|
||||||
|
# 7-day-ahead: every feature is at least 7 days old, so a 168 h forecast is
|
||||||
|
# possible without recursion.
|
||||||
|
FEATURE_WEEK = FEATURE_CALENDAR + ["lag_week", "seasonal_mean_4w"]
|
||||||
|
FEATURE_WEATHER = ["temp_c", "precip_mm"]
|
||||||
|
|
||||||
|
FEATURE_DAY_NO_HOLIDAY = FEATURE_CALENDAR_NO_HOLIDAY + FEATURE_DAY[6:]
|
||||||
|
FEATURE_DAY_NO_WEATHER = FEATURE_DAY
|
||||||
|
FEATURE_DAY_WEATHER = FEATURE_DAY + FEATURE_WEATHER
|
||||||
|
FEATURE_DAY_HOLIDAY = FEATURE_DAY + ["is_holiday"] # is_holiday already present; kept for clarity
|
||||||
|
FEATURE_WEEK_NO_WEATHER = FEATURE_WEEK
|
||||||
|
FEATURE_WEEK_WEATHER = FEATURE_WEEK + FEATURE_WEATHER
|
||||||
|
|
||||||
|
# Scenario presets. Weather adjustments apply on the event's weekdays; the
|
||||||
|
# event uplift is a transparent multiplicative overlay applied after the model.
|
||||||
|
# These are decision-support assumptions, not causal estimates.
|
||||||
|
SCENARIOS: dict[str, dict] = {
|
||||||
|
"base": {
|
||||||
|
"label": "Base forecast",
|
||||||
|
"temp_delta_c": 0.0,
|
||||||
|
"precip_mm": None,
|
||||||
|
"event": None,
|
||||||
|
},
|
||||||
|
"sunny-saturday-market": {
|
||||||
|
"label": "Sunny Saturday + Christmas market",
|
||||||
|
"temp_delta_c": 4.0,
|
||||||
|
"precip_mm": 0.0,
|
||||||
|
"event": {"type": "market", "uplift": 0.15, "weekdays": [5], "hours": list(range(10, 21))},
|
||||||
|
},
|
||||||
|
"rainy-saturday": {
|
||||||
|
"label": "Rainy Saturday",
|
||||||
|
"temp_delta_c": -3.0,
|
||||||
|
"precip_mm": 4.0,
|
||||||
|
"event": {"type": "rain", "uplift": -0.15, "weekdays": [5], "hours": list(range(10, 21))},
|
||||||
|
},
|
||||||
|
"weekday-event": {
|
||||||
|
"label": "Weekday evening event",
|
||||||
|
"temp_delta_c": 0.0,
|
||||||
|
"precip_mm": None,
|
||||||
|
"event": {"type": "event", "uplift": 0.10, "weekdays": [0, 1, 2, 3, 4], "hours": [17, 18, 19, 20]},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
HORIZONS = ("15m", "1h", "48h", "7d")
|
||||||
|
HORIZON_STEPS = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
|
||||||
|
|
||||||
|
LIMITS = [
|
||||||
|
"estimated devices, not people",
|
||||||
|
"8 sensors in a ~580 m inner-city cluster",
|
||||||
|
"5 months of multi-sensor data; no annual seasonality claim",
|
||||||
|
"observed weather in training, forecast weather in scenarios",
|
||||||
|
"holidays are context, not a large accuracy claim",
|
||||||
|
]
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
"""Hourly aggregation and leakage-safe feature construction.
|
||||||
|
|
||||||
|
Design rules:
|
||||||
|
- Hour bins are UTC hours (unambiguous across DST); local calendar fields are
|
||||||
|
derived from Europe/Berlin.
|
||||||
|
- Every lag feature is at least 24 h old (day-ahead) or 168 h old (7-day).
|
||||||
|
Nothing at hour t uses the target at hour t or later.
|
||||||
|
- Seasonal lags use the previous *observed* same weekday/hour, so sensor gaps
|
||||||
|
do not silently break the feature.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
SENSOR_CODES = {name: i for i, name in enumerate(config.USABLE_SENSORS)}
|
||||||
|
|
||||||
|
|
||||||
|
def to_hourly(density: pd.DataFrame) -> pd.DataFrame:
|
||||||
|
"""Aggregate 15-minute density readings to UTC hourly means per sensor."""
|
||||||
|
df = density.copy()
|
||||||
|
df["t"] = df["time"].dt.floor("h")
|
||||||
|
hourly = (
|
||||||
|
df.groupby(["sensor", "t"], observed=True)
|
||||||
|
.agg(v=("value", "mean"), n=("value", "size"))
|
||||||
|
.reset_index()
|
||||||
|
)
|
||||||
|
hourly["v"] = hourly["v"].astype(float)
|
||||||
|
return hourly.sort_values(["sensor", "t"]).reset_index(drop=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _calendar(frame: pd.DataFrame, time_col: str = "t") -> pd.DataFrame:
|
||||||
|
local = frame[time_col].dt.tz_convert(config.TIMEZONE)
|
||||||
|
out = frame.copy()
|
||||||
|
out["local_date"] = local.dt.strftime("%Y-%m-%d")
|
||||||
|
out["hour"] = local.dt.hour.astype("int8")
|
||||||
|
out["weekday"] = local.dt.day_of_week.astype("int8")
|
||||||
|
out["month"] = local.dt.month.astype("int8")
|
||||||
|
out["is_weekend"] = (out["weekday"] >= 5).astype("int8")
|
||||||
|
out["holiday_name"] = out["local_date"].map(config.HOLIDAYS)
|
||||||
|
out["is_holiday"] = out["holiday_name"].notna().astype("int8")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _grid(hourly: pd.DataFrame, end: pd.Timestamp) -> pd.DataFrame:
|
||||||
|
"""Continuous hourly grid per sensor; used only for time-based rollups."""
|
||||||
|
parts = []
|
||||||
|
for sensor, group in hourly.groupby("sensor", observed=True):
|
||||||
|
idx = pd.date_range(group["t"].min(), end, freq="h", tz="UTC")
|
||||||
|
s = group.set_index("t")["v"].reindex(idx)
|
||||||
|
part = pd.DataFrame({"sensor": sensor, "t": idx, "y": s.to_numpy()})
|
||||||
|
parts.append(part)
|
||||||
|
grid = pd.concat(parts, ignore_index=True)
|
||||||
|
grid = grid.sort_values(["sensor", "t"]).reset_index(drop=True)
|
||||||
|
return grid
|
||||||
|
|
||||||
|
|
||||||
|
def _add_rollups(grid: pd.DataFrame) -> pd.DataFrame:
|
||||||
|
"""Rolling means shifted 24 h back: only yesterday-and-older information."""
|
||||||
|
out = []
|
||||||
|
for _, group in grid.groupby("sensor", observed=True):
|
||||||
|
g = group.sort_values("t").copy()
|
||||||
|
shifted = g["y"].shift(24)
|
||||||
|
g["roll_24"] = shifted.rolling(24, min_periods=12).mean()
|
||||||
|
g["roll_168"] = shifted.rolling(168, min_periods=84).mean()
|
||||||
|
out.append(g)
|
||||||
|
return pd.concat(out, ignore_index=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _add_seasonal_lags(df: pd.DataFrame) -> pd.DataFrame:
|
||||||
|
"""Previous observed same weekday/hour values, strictly in the past."""
|
||||||
|
out = df.sort_values(["sensor", "weekday", "hour", "t"]).copy()
|
||||||
|
grouped = out.groupby(["sensor", "weekday", "hour"], observed=True)["y"]
|
||||||
|
out["lag_week"] = grouped.shift(1)
|
||||||
|
prior = pd.concat(
|
||||||
|
[grouped.shift(k) for k in (2, 3, 4)],
|
||||||
|
axis=1,
|
||||||
|
)
|
||||||
|
prior.columns = ["lag_2w", "lag_3w", "lag_4w"]
|
||||||
|
all_prior = pd.concat([out["lag_week"], prior], axis=1)
|
||||||
|
counts = all_prior.notna().sum(axis=1)
|
||||||
|
out["seasonal_mean_4w"] = all_prior.mean(axis=1, skipna=True).where(counts >= 2)
|
||||||
|
|
||||||
|
by_hour = out.groupby(["sensor", "hour"], observed=True)["y"]
|
||||||
|
out["lag_day"] = by_hour.shift(1)
|
||||||
|
out["lag_2d"] = by_hour.shift(2)
|
||||||
|
return out.sort_values(["sensor", "t"]).reset_index(drop=True)
|
||||||
|
|
||||||
|
|
||||||
|
def build_features(
|
||||||
|
hourly: pd.DataFrame,
|
||||||
|
weather: pd.DataFrame,
|
||||||
|
*,
|
||||||
|
data_end: pd.Timestamp | None = None,
|
||||||
|
) -> pd.DataFrame:
|
||||||
|
"""Build the full feature table (observed hours only, target present)."""
|
||||||
|
hourly = hourly[hourly["v"].notna()].copy()
|
||||||
|
end = data_end or hourly["t"].max()
|
||||||
|
grid = _add_rollups(_grid(hourly, end))
|
||||||
|
df = hourly.merge(
|
||||||
|
grid[["sensor", "t", "y", "roll_24", "roll_168"]],
|
||||||
|
on=["sensor", "t"],
|
||||||
|
how="left",
|
||||||
|
)
|
||||||
|
df = _calendar(df)
|
||||||
|
df["sensor_code"] = df["sensor"].map(SENSOR_CODES).astype("int16")
|
||||||
|
df = _add_seasonal_lags(df)
|
||||||
|
df = df.merge(weather.drop(columns=[c for c in ("time",) if c in weather.columns]).assign(
|
||||||
|
t=weather["time"]
|
||||||
|
), on="t", how="left")
|
||||||
|
df = df[df["y"].notna()].copy()
|
||||||
|
return df.sort_values(["sensor", "t"]).reset_index(drop=True)
|
||||||
|
|
||||||
|
|
||||||
|
def feature_columns(kind: str) -> list[str]:
|
||||||
|
mapping = {
|
||||||
|
"day_no_holiday": config.FEATURE_DAY_NO_HOLIDAY,
|
||||||
|
"day": config.FEATURE_DAY,
|
||||||
|
"day_weather": config.FEATURE_DAY_WEATHER,
|
||||||
|
"week": config.FEATURE_WEEK,
|
||||||
|
"week_weather": config.FEATURE_WEEK_WEATHER,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
return list(mapping[kind])
|
||||||
|
except KeyError as exc:
|
||||||
|
raise ValueError(f"unknown feature set {kind!r}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def seasonal_lookup(hourly: pd.DataFrame) -> dict[str, dict[str, float]]:
|
||||||
|
"""Last-4-observation lookup per sensor/weekday/hour for future inference.
|
||||||
|
|
||||||
|
Key format: "<sensor>|<weekday>|<hour>" -> {"mean4": float, "last": float}.
|
||||||
|
This is the same definition used by ``lag_week`` / ``seasonal_mean_4w`` in
|
||||||
|
training, evaluated at the last observed week, so a future timestamp can use
|
||||||
|
it without assuming data after the CSV end.
|
||||||
|
"""
|
||||||
|
df = _calendar(hourly[hourly["v"].notna()].copy())
|
||||||
|
lookup: dict[str, dict[str, float]] = {}
|
||||||
|
for (sensor, weekday, hour), group in df.groupby(
|
||||||
|
["sensor", "weekday", "hour"], observed=True
|
||||||
|
):
|
||||||
|
last4 = group.sort_values("t")["v"].tail(4)
|
||||||
|
lookup[f"{sensor}|{int(weekday)}|{int(hour)}"] = {
|
||||||
|
"mean4": round(float(last4.mean()), 4),
|
||||||
|
"last": round(float(last4.iloc[-1]), 4),
|
||||||
|
}
|
||||||
|
return lookup
|
||||||
|
|
||||||
|
|
||||||
|
def future_calendar(start: pd.Timestamp, periods: int) -> pd.DataFrame:
|
||||||
|
"""Calendar features for a future hourly grid (no target, no lags)."""
|
||||||
|
t = pd.date_range(start, periods=periods, freq="h", tz="UTC")
|
||||||
|
frame = pd.DataFrame({"t": t})
|
||||||
|
frame = _calendar(frame)
|
||||||
|
return frame
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
"""Parse and clean the raw PAX CSV.
|
||||||
|
|
||||||
|
Verifies the claims in mission/data.md: 192,464 rows, 12 device IDs, 8 usable
|
||||||
|
logical locations, three excluded test devices, and the density-01 merge.
|
||||||
|
The raw file is never written to git; a cached parquet copy lives in data/cache/.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
EXPECTED_CHECKS = {
|
||||||
|
"raw_rows": 192_464,
|
||||||
|
"density_rows": 96_232,
|
||||||
|
"battery_rows": 96_232,
|
||||||
|
"device_ids": 12,
|
||||||
|
"sensor_names": 11,
|
||||||
|
"usable_sensors": 8,
|
||||||
|
"density_01_rows_multi_sensor": 12_670,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _logical_sensor(name: str, device_id: str) -> str:
|
||||||
|
return config.DEVICE_TO_SENSOR.get(device_id, name)
|
||||||
|
|
||||||
|
|
||||||
|
def load_raw(path: Path | None = None) -> pd.DataFrame:
|
||||||
|
path = path or config.raw_csv_path()
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"raw CSV not found at {path}. Set PULSECAST_RAW_CSV or place the "
|
||||||
|
"challenge download at that path (never commit it)."
|
||||||
|
)
|
||||||
|
df = pd.read_csv(path, parse_dates=["time"], dtype={"sensor_id": "string", "name": "string"})
|
||||||
|
df["time"] = pd.to_datetime(df["time"], utc=True)
|
||||||
|
expected = {"time", "sensor_id", "name", "longitude", "latitude", "type", "value"}
|
||||||
|
missing = expected - set(df.columns)
|
||||||
|
if missing:
|
||||||
|
raise ValueError(f"raw CSV missing columns: {sorted(missing)}")
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def clean(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
||||||
|
"""Return (density, battery, audit) with logical sensors and exclusions applied."""
|
||||||
|
density = df[df["type"] == "densityData"].copy()
|
||||||
|
battery = df[df["type"] == "batteryData"].copy()
|
||||||
|
|
||||||
|
for frame in (density, battery):
|
||||||
|
frame["sensor"] = [
|
||||||
|
_logical_sensor(n, d) for n, d in zip(frame["name"], frame["sensor_id"], strict=True)
|
||||||
|
]
|
||||||
|
|
||||||
|
excluded_rows = {
|
||||||
|
name: int((density["sensor"] == name).sum())
|
||||||
|
for name in config.EXCLUDED_SENSORS
|
||||||
|
}
|
||||||
|
density = density[~density["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
|
||||||
|
battery = battery[~battery["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
|
||||||
|
|
||||||
|
before = len(density)
|
||||||
|
density = density[
|
||||||
|
density["value"].between(config.VALUE_MIN, config.VALUE_MAX, inclusive="both")
|
||||||
|
].copy()
|
||||||
|
dropped_out_of_range = before - len(density)
|
||||||
|
|
||||||
|
usable = sorted(density["sensor"].unique().tolist())
|
||||||
|
if set(usable) != set(config.USABLE_SENSORS):
|
||||||
|
raise ValueError(
|
||||||
|
f"usable sensor set changed: {usable}; expected {sorted(config.USABLE_SENSORS)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
|
||||||
|
density_01_devices = (
|
||||||
|
multi[multi["sensor"] == "density-01"]
|
||||||
|
.groupby("sensor_id", observed=True)
|
||||||
|
.agg(rows=("value", "size"), first=("time", "min"), last=("time", "max"))
|
||||||
|
)
|
||||||
|
|
||||||
|
# One reading per (time, device, type); duplicates would double-weight a bin.
|
||||||
|
dup_keys = int(density.duplicated(subset=["time", "sensor_id"]).sum())
|
||||||
|
if dup_keys:
|
||||||
|
density = density.drop_duplicates(subset=["time", "sensor_id"], keep="last")
|
||||||
|
|
||||||
|
cadence = (
|
||||||
|
density.sort_values(["sensor", "time"])
|
||||||
|
.groupby("sensor", observed=True)["time"]
|
||||||
|
.diff()
|
||||||
|
.dt.total_seconds()
|
||||||
|
.div(60)
|
||||||
|
.dropna()
|
||||||
|
)
|
||||||
|
|
||||||
|
audit = {
|
||||||
|
"raw_rows": int(len(df)),
|
||||||
|
"density_rows": int(len(df[df["type"] == "densityData"])),
|
||||||
|
"battery_rows": int(len(df[df["type"] == "batteryData"])),
|
||||||
|
"device_ids": int(df["sensor_id"].nunique()),
|
||||||
|
"sensor_names": int(df["name"].nunique()),
|
||||||
|
"usable_sensors": len(usable),
|
||||||
|
"sensor_ids": usable,
|
||||||
|
"excluded_sensors": {
|
||||||
|
name: {"rows": excluded_rows[name], "reason": reason}
|
||||||
|
for name, reason in config.EXCLUDED_SENSORS.items()
|
||||||
|
},
|
||||||
|
"density_01_devices": {
|
||||||
|
str(device): {
|
||||||
|
"rows": int(row["rows"]),
|
||||||
|
"first": _iso(row["first"]),
|
||||||
|
"last": _iso(row["last"]),
|
||||||
|
}
|
||||||
|
for device, row in density_01_devices.iterrows()
|
||||||
|
},
|
||||||
|
"density_01_rows_multi_sensor": int(
|
||||||
|
(multi["sensor"] == "density-01").sum()
|
||||||
|
),
|
||||||
|
"time_range": {
|
||||||
|
"from": _iso(density["time"].min()),
|
||||||
|
"to": _iso(density["time"].max()),
|
||||||
|
},
|
||||||
|
"multi_sensor_from": _iso(multi["time"].min()),
|
||||||
|
"cadence_median_min": round(float(cadence.median()), 2),
|
||||||
|
"dropped_out_of_range": int(dropped_out_of_range),
|
||||||
|
"duplicate_keys_dropped": dup_keys,
|
||||||
|
"checks": [],
|
||||||
|
"all_checks_ok": True,
|
||||||
|
}
|
||||||
|
for key, expected in EXPECTED_CHECKS.items():
|
||||||
|
actual = audit[key]
|
||||||
|
ok = int(actual) == int(expected)
|
||||||
|
audit["checks"].append({"name": key, "expected": expected, "actual": actual, "ok": ok})
|
||||||
|
if not ok:
|
||||||
|
audit["all_checks_ok"] = False
|
||||||
|
log.warning("data audit mismatch %s: expected %s, got %s", key, expected, actual)
|
||||||
|
return density, battery, audit
|
||||||
|
|
||||||
|
|
||||||
|
def write_audit(audit: dict, path: Path | None = None) -> Path:
|
||||||
|
path = path or config.artifacts_dir() / "data_audit.json"
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(audit, indent=2, ensure_ascii=False) + "\n")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def load_cached_density() -> pd.DataFrame:
|
||||||
|
path = config.cache_dir() / "density_clean.parquet"
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(f"{path} missing; run `make data` first")
|
||||||
|
return pd.read_parquet(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(ts) -> str | None:
|
||||||
|
if ts is None or pd.isna(ts):
|
||||||
|
return None
|
||||||
|
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
"""Open-Meteo weather: historical archive for training, forecast for scenarios.
|
||||||
|
|
||||||
|
No API key. Responses are cached under data/cache/ so `make train` and the
|
||||||
|
Docker image need no network. The raw CSV and cache stay out of git; the final
|
||||||
|
context.json is committed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from . import config
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive"
|
||||||
|
FORECAST_URL = "https://api.open-meteo.com/v1/forecast"
|
||||||
|
ARCHIVE_FILE = "weather_archive.parquet"
|
||||||
|
FORECAST_FILE = "weather_forecast.parquet"
|
||||||
|
META_FILE = "weather_meta.json"
|
||||||
|
HOURLY_PARAMS = ",".join(config.WEATHER_FIELDS)
|
||||||
|
CURRENT_PARAMS = HOURLY_PARAMS
|
||||||
|
|
||||||
|
|
||||||
|
def _get_json(url: str, params: dict, *, retries: int = 3) -> dict:
|
||||||
|
last: Exception | None = None
|
||||||
|
for attempt in range(retries):
|
||||||
|
try:
|
||||||
|
with httpx.Client(timeout=30.0) as client:
|
||||||
|
resp = client.get(url, params=params)
|
||||||
|
resp.raise_for_status()
|
||||||
|
payload = resp.json()
|
||||||
|
if payload.get("error"):
|
||||||
|
raise RuntimeError(f"Open-Meteo error: {payload.get('reason', payload)}")
|
||||||
|
return payload
|
||||||
|
except Exception as exc: # network / HTTP / payload
|
||||||
|
last = exc
|
||||||
|
if attempt < retries - 1:
|
||||||
|
time.sleep(2**attempt)
|
||||||
|
raise RuntimeError(f"Open-Meteo request failed after {retries} attempts: {last}")
|
||||||
|
|
||||||
|
|
||||||
|
def _hourly_frame(payload: dict) -> pd.DataFrame:
|
||||||
|
hourly = payload.get("hourly") or {}
|
||||||
|
if "time" not in hourly:
|
||||||
|
raise RuntimeError("Open-Meteo payload has no hourly.time")
|
||||||
|
frame = pd.DataFrame(hourly)
|
||||||
|
frame["time"] = pd.to_datetime(frame["time"], utc=True)
|
||||||
|
missing = [f for f in config.WEATHER_FIELDS if f not in frame.columns]
|
||||||
|
if missing:
|
||||||
|
raise RuntimeError(f"Open-Meteo payload missing fields: {missing}")
|
||||||
|
frame = frame.rename(columns=config.WEATHER_COLUMNS)
|
||||||
|
cols = ["time"] + list(config.WEATHER_COLUMNS.values())
|
||||||
|
return frame[cols]
|
||||||
|
|
||||||
|
|
||||||
|
def _meta_path() -> Path:
|
||||||
|
return config.cache_dir() / META_FILE
|
||||||
|
|
||||||
|
|
||||||
|
def _load_meta() -> dict:
|
||||||
|
path = _meta_path()
|
||||||
|
if path.exists():
|
||||||
|
return json.loads(path.read_text())
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_archive(start: date, end: date) -> pd.DataFrame:
|
||||||
|
payload = _get_json(
|
||||||
|
ARCHIVE_URL,
|
||||||
|
{
|
||||||
|
"latitude": config.LATITUDE,
|
||||||
|
"longitude": config.LONGITUDE,
|
||||||
|
"start_date": start.isoformat(),
|
||||||
|
"end_date": end.isoformat(),
|
||||||
|
"hourly": HOURLY_PARAMS,
|
||||||
|
"timezone": "UTC",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
frame = _hourly_frame(payload)
|
||||||
|
log.info("fetched %s archive weather rows for %s..%s", len(frame), start, end)
|
||||||
|
return frame
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_forecast(days: int = 8) -> tuple[pd.DataFrame, dict, dict]:
|
||||||
|
payload = _get_json(
|
||||||
|
FORECAST_URL,
|
||||||
|
{
|
||||||
|
"latitude": config.LATITUDE,
|
||||||
|
"longitude": config.LONGITUDE,
|
||||||
|
"hourly": HOURLY_PARAMS,
|
||||||
|
"current": CURRENT_PARAMS,
|
||||||
|
"forecast_days": days,
|
||||||
|
"timezone": "UTC",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
frame = _hourly_frame(payload)
|
||||||
|
current = payload.get("current") or {}
|
||||||
|
now = {
|
||||||
|
"t": current.get("time"),
|
||||||
|
"temp_c": current.get("temperature_2m"),
|
||||||
|
"precip_mm": current.get("precipitation"),
|
||||||
|
"cloud_pct": current.get("cloud_cover"),
|
||||||
|
"wind_kmh": current.get("wind_speed_10m"),
|
||||||
|
"humidity_pct": current.get("relative_humidity_2m"),
|
||||||
|
}
|
||||||
|
grid = {"latitude": payload.get("latitude"), "longitude": payload.get("longitude")}
|
||||||
|
meta = {
|
||||||
|
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
"current": now,
|
||||||
|
"grid": grid,
|
||||||
|
"source": "open-meteo",
|
||||||
|
}
|
||||||
|
log.info("fetched %s forecast weather rows", len(frame))
|
||||||
|
return frame, meta, grid
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_weather(
|
||||||
|
data_start: pd.Timestamp | None = None,
|
||||||
|
data_end: pd.Timestamp | None = None,
|
||||||
|
*,
|
||||||
|
refresh: bool = False,
|
||||||
|
) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
||||||
|
"""Load cached weather, fetching only when missing, stale, or refresh=True."""
|
||||||
|
cache = config.cache_dir()
|
||||||
|
cache.mkdir(parents=True, exist_ok=True)
|
||||||
|
archive_path = cache / ARCHIVE_FILE
|
||||||
|
forecast_path = cache / FORECAST_FILE
|
||||||
|
meta = _load_meta()
|
||||||
|
|
||||||
|
if archive_path.exists() and not refresh:
|
||||||
|
archive = pd.read_parquet(archive_path)
|
||||||
|
else:
|
||||||
|
start = (data_start or pd.Timestamp(config.MULTI_SENSOR_START)).date() - timedelta(days=2)
|
||||||
|
end = (data_end or pd.Timestamp.now(tz="UTC")).date() + timedelta(days=1)
|
||||||
|
archive = fetch_archive(start, end)
|
||||||
|
archive.to_parquet(archive_path, index=False)
|
||||||
|
|
||||||
|
if forecast_path.exists() and not refresh:
|
||||||
|
forecast = pd.read_parquet(forecast_path)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
forecast, meta, _ = fetch_forecast()
|
||||||
|
forecast.to_parquet(forecast_path, index=False)
|
||||||
|
_meta_path().write_text(json.dumps(meta, indent=2) + "\n")
|
||||||
|
except Exception:
|
||||||
|
if not forecast_path.exists():
|
||||||
|
raise
|
||||||
|
log.warning("Open-Meteo forecast refresh failed; using cached forecast")
|
||||||
|
forecast = pd.read_parquet(forecast_path)
|
||||||
|
|
||||||
|
if not meta:
|
||||||
|
meta = {
|
||||||
|
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
|
"current": None,
|
||||||
|
"grid": {"latitude": config.LATITUDE, "longitude": config.LONGITUDE},
|
||||||
|
"source": "cache",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Keep the archive within the challenge window (+/- 2 days) to avoid
|
||||||
|
# accidentally training on future weather if the cache is refreshed later.
|
||||||
|
if data_end is not None:
|
||||||
|
archive = archive[archive["time"] <= data_end + pd.Timedelta(days=1)].copy()
|
||||||
|
if data_start is not None:
|
||||||
|
archive = archive[archive["time"] >= data_start - pd.Timedelta(days=2)].copy()
|
||||||
|
return archive.reset_index(drop=True), forecast.reset_index(drop=True), meta
|
||||||
|
|
||||||
|
|
||||||
|
def load_cached_weather() -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
||||||
|
cache = config.cache_dir()
|
||||||
|
archive = pd.read_parquet(cache / ARCHIVE_FILE)
|
||||||
|
forecast = pd.read_parquet(cache / FORECAST_FILE)
|
||||||
|
return archive, forecast, _load_meta()
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_weather() -> None:
|
||||||
|
archive, forecast, meta = ensure_weather(refresh=True)
|
||||||
|
log.info("weather refreshed: archive=%s forecast=%s", len(archive), len(forecast))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||||
|
refresh_weather()
|
||||||
Reference in new issue
Block a user