chore: bootstrap ML-py scaffold

This commit is contained in:
Noel Sabia committed 2026-11-14 04:16:22 +01:00
commit 2eeb1f937a
12 files changed
+1924

No files matched your search

+23
View File
@@ -0,0 +1,23 @@
# Raw challenge data must never be committed.
data/raw/
data/cache/
# Python
__pycache__/
*.py[cod]
.venv/
.pytest_cache/
.ruff_cache/
*.egg-info/
dist/
build/
# OS / editor
.DS_Store
.idea/
.vscode/
# Local secrets / kubeconfigs
*.kubeconfig
kubeconfig*.yaml
.env
+1
View File
@@ -0,0 +1 @@
3.12
View File
Whitespace-only changes.
+6
View File
@@ -0,0 +1,6 @@
def main():
print("Hello from ml-py!")
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
{
"$schema": "https://opencode.ai/config.json",
"model": "opencode-go/deepseek-v4.1-flash"
}
+31
View File
@@ -0,0 +1,31 @@
[project]
name = "pulsecast-ml"
version = "0.1.0"
description = "PulseCast ML: footfall pipeline, weather-aware forecast, scenario engine, FastAPI service"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"fastapi>=0.162.7",
"httpx>=0.28.1",
"joblib>=1.6.0",
"numpy>=2.5.4",
"pandas>=3.1.1",
"pyarrow>=26.0.1",
"pytest>=9.2.0",
"scikit-learn>=1.9.1",
"uvicorn[standard]>=0.54.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/pulsecast"]
[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "-q"
[dependency-groups]
dev = []
+3
View File
@@ -0,0 +1,3 @@
"""PulseCast ML package."""
__version__ = "0.1.0"
+213
View File
@@ -0,0 +1,213 @@
"""Configuration: paths, sensors, holidays, scenarios, contract versions.
All timestamps are UTC internally; Europe/Berlin is used only for calendar
features and display-oriented payloads.
"""
from __future__ import annotations
import os
from pathlib import Path
# --------------------------------------------------------------------------- paths
def _root() -> Path:
"""Project root: PULSECAST_ROOT, else the first parent containing pyproject.toml.
Works both from a checkout (src/pulsecast/config.py -> repo root) and from
``python -m`` / pytest launched at the repo root.
"""
env = os.environ.get("PULSECAST_ROOT")
if env:
return Path(env).resolve()
here = Path(__file__).resolve()
for parent in here.parents:
if (parent / "pyproject.toml").exists():
return parent
return Path.cwd()
def artifacts_dir() -> Path:
return Path(os.environ.get("PULSECAST_ARTIFACTS", _root() / "artifacts")).resolve()
def cache_dir() -> Path:
return Path(os.environ.get("PULSECAST_CACHE", _root() / "data" / "cache")).resolve()
def raw_csv_path() -> Path:
return Path(
os.environ.get(
"PULSECAST_RAW_CSV",
"/home/noel/Documents/Hackathon_2026/data/raw/schwarz-pax.csv",
)
)
# --------------------------------------------------------------------------- data
# Multi-sensor window documented in mission/data.md. density-01 has older
# history but the other seven locations start on 2026-06-12.
MULTI_SENSOR_START = "2026-06-12T00:00:00Z"
# 28-day time-based holdout, measured back from the last observed hour.
TEST_DAYS = 28
# Excluded devices, with the reason recorded in the data audit.
EXCLUDED_SENSORS: dict[str, str] = {
"maker-fair": "test installation in Hannover, 73 readings, outside the inner city",
"niklas-01": "8 readings, outside the inner-city cluster",
"hn-bollwerksturm-01": "5 readings, insufficient coverage",
}
# Two physical device IDs at one location; merge into one logical sensor.
DEVICE_TO_SENSOR: dict[str, str] = {
"863f75b0": "density-01",
"f8c19730": "density-01",
}
USABLE_SENSORS: tuple[str, ...] = (
"density-01",
"hn-goetzentuem-01",
"hn-hagenbuchersee-01",
"hn-hagenbuchersee-02",
"hn-marahaus-01",
"hn-neckarufer-01",
"hn-soleo-01",
"hn-stadtgalerie-01",
)
DISPLAY_NAMES: dict[str, str] = {
"density-01": "Density 01",
"hn-goetzentuem-01": "Götzenturm",
"hn-hagenbuchersee-01": "Hagenbuchersee 1",
"hn-hagenbuchersee-02": "Hagenbuchersee 2",
"hn-marahaus-01": "Marahaus",
"hn-neckarufer-01": "Neckarufer",
"hn-soleo-01": "Soleo",
"hn-stadtgalerie-01": "Stadtgalerie",
}
# Reject rows outside this plausible range before aggregation.
VALUE_MIN = 0.0
VALUE_MAX = 1000.0
# --------------------------------------------------------------------------- context
# Heilbronn inner city (Stadtgalerie / Kaiserstraße area). Open-Meteo snaps to
# its nearest grid cell; document the returned coordinates in context.json.
LATITUDE = 49.1427
LONGITUDE = 9.2109
TIMEZONE = "Europe/Berlin"
WEATHER_FIELDS = (
"temperature_2m",
"precipitation",
"cloud_cover",
"wind_speed_10m",
"relative_humidity_2m",
)
WEATHER_COLUMNS = {
"temperature_2m": "temp_c",
"precipitation": "precip_mm",
"cloud_cover": "cloud_pct",
"wind_speed_10m": "wind_kmh",
"relative_humidity_2m": "humidity_pct",
}
# Baden-Württemberg public holidays, 2025-2026 (mission/data.md). Kept explicit
# so the build is deterministic and needs no holiday library.
HOLIDAYS: dict[str, str] = {
"2025-01-01": "Neujahr",
"2025-01-06": "Heilige Drei Könige",
"2025-04-18": "Karfreitag",
"2025-04-21": "Ostermontag",
"2025-05-01": "Tag der Arbeit",
"2025-05-29": "Christi Himmelfahrt",
"2025-06-09": "Pfingstmontag",
"2025-06-19": "Fronleichnam",
"2025-10-03": "Tag der Deutschen Einheit",
"2025-11-01": "Allerheiligen",
"2025-12-25": "1. Weihnachtstag",
"2025-12-26": "2. Weihnachtstag",
"2026-01-01": "Neujahr",
"2026-01-06": "Heilige Drei Könige",
"2026-04-03": "Karfreitag",
"2026-04-06": "Ostermontag",
"2026-05-01": "Tag der Arbeit",
"2026-05-14": "Christi Himmelfahrt",
"2026-05-25": "Pfingstmontag",
"2026-06-04": "Fronleichnam",
"2026-10-03": "Tag der Deutschen Einheit",
"2026-11-01": "Allerheiligen",
"2026-12-25": "1. Weihnachtstag",
"2026-12-26": "2. Weihnachtstag",
}
# --------------------------------------------------------------------------- model
FEATURE_CALENDAR = ["hour", "weekday", "month", "is_weekend", "is_holiday", "sensor_code"]
FEATURE_CALENDAR_NO_HOLIDAY = ["hour", "weekday", "month", "is_weekend", "sensor_code"]
# Day-ahead: every feature is at least 24 h old.
FEATURE_DAY = FEATURE_CALENDAR + [
"lag_day",
"lag_2d",
"lag_week",
"seasonal_mean_4w",
"roll_24",
"roll_168",
]
# 7-day-ahead: every feature is at least 7 days old, so a 168 h forecast is
# possible without recursion.
FEATURE_WEEK = FEATURE_CALENDAR + ["lag_week", "seasonal_mean_4w"]
FEATURE_WEATHER = ["temp_c", "precip_mm"]
FEATURE_DAY_NO_HOLIDAY = FEATURE_CALENDAR_NO_HOLIDAY + FEATURE_DAY[6:]
FEATURE_DAY_NO_WEATHER = FEATURE_DAY
FEATURE_DAY_WEATHER = FEATURE_DAY + FEATURE_WEATHER
FEATURE_DAY_HOLIDAY = FEATURE_DAY + ["is_holiday"] # is_holiday already present; kept for clarity
FEATURE_WEEK_NO_WEATHER = FEATURE_WEEK
FEATURE_WEEK_WEATHER = FEATURE_WEEK + FEATURE_WEATHER
# Scenario presets. Weather adjustments apply on the event's weekdays; the
# event uplift is a transparent multiplicative overlay applied after the model.
# These are decision-support assumptions, not causal estimates.
SCENARIOS: dict[str, dict] = {
"base": {
"label": "Base forecast",
"temp_delta_c": 0.0,
"precip_mm": None,
"event": None,
},
"sunny-saturday-market": {
"label": "Sunny Saturday + Christmas market",
"temp_delta_c": 4.0,
"precip_mm": 0.0,
"event": {"type": "market", "uplift": 0.15, "weekdays": [5], "hours": list(range(10, 21))},
},
"rainy-saturday": {
"label": "Rainy Saturday",
"temp_delta_c": -3.0,
"precip_mm": 4.0,
"event": {"type": "rain", "uplift": -0.15, "weekdays": [5], "hours": list(range(10, 21))},
},
"weekday-event": {
"label": "Weekday evening event",
"temp_delta_c": 0.0,
"precip_mm": None,
"event": {"type": "event", "uplift": 0.10, "weekdays": [0, 1, 2, 3, 4], "hours": [17, 18, 19, 20]},
},
}
HORIZONS = ("15m", "1h", "48h", "7d")
HORIZON_STEPS = {"15m": 1, "1h": 4, "48h": 48, "7d": 168}
LIMITS = [
"estimated devices, not people",
"8 sensors in a ~580 m inner-city cluster",
"5 months of multi-sensor data; no annual seasonality claim",
"observed weather in training, forecast weather in scenarios",
"holidays are context, not a large accuracy claim",
]
+158
View File
@@ -0,0 +1,158 @@
"""Hourly aggregation and leakage-safe feature construction.
Design rules:
- Hour bins are UTC hours (unambiguous across DST); local calendar fields are
derived from Europe/Berlin.
- Every lag feature is at least 24 h old (day-ahead) or 168 h old (7-day).
Nothing at hour t uses the target at hour t or later.
- Seasonal lags use the previous *observed* same weekday/hour, so sensor gaps
do not silently break the feature.
"""
from __future__ import annotations
import numpy as np
import pandas as pd
from . import config
SENSOR_CODES = {name: i for i, name in enumerate(config.USABLE_SENSORS)}
def to_hourly(density: pd.DataFrame) -> pd.DataFrame:
"""Aggregate 15-minute density readings to UTC hourly means per sensor."""
df = density.copy()
df["t"] = df["time"].dt.floor("h")
hourly = (
df.groupby(["sensor", "t"], observed=True)
.agg(v=("value", "mean"), n=("value", "size"))
.reset_index()
)
hourly["v"] = hourly["v"].astype(float)
return hourly.sort_values(["sensor", "t"]).reset_index(drop=True)
def _calendar(frame: pd.DataFrame, time_col: str = "t") -> pd.DataFrame:
local = frame[time_col].dt.tz_convert(config.TIMEZONE)
out = frame.copy()
out["local_date"] = local.dt.strftime("%Y-%m-%d")
out["hour"] = local.dt.hour.astype("int8")
out["weekday"] = local.dt.day_of_week.astype("int8")
out["month"] = local.dt.month.astype("int8")
out["is_weekend"] = (out["weekday"] >= 5).astype("int8")
out["holiday_name"] = out["local_date"].map(config.HOLIDAYS)
out["is_holiday"] = out["holiday_name"].notna().astype("int8")
return out
def _grid(hourly: pd.DataFrame, end: pd.Timestamp) -> pd.DataFrame:
"""Continuous hourly grid per sensor; used only for time-based rollups."""
parts = []
for sensor, group in hourly.groupby("sensor", observed=True):
idx = pd.date_range(group["t"].min(), end, freq="h", tz="UTC")
s = group.set_index("t")["v"].reindex(idx)
part = pd.DataFrame({"sensor": sensor, "t": idx, "y": s.to_numpy()})
parts.append(part)
grid = pd.concat(parts, ignore_index=True)
grid = grid.sort_values(["sensor", "t"]).reset_index(drop=True)
return grid
def _add_rollups(grid: pd.DataFrame) -> pd.DataFrame:
"""Rolling means shifted 24 h back: only yesterday-and-older information."""
out = []
for _, group in grid.groupby("sensor", observed=True):
g = group.sort_values("t").copy()
shifted = g["y"].shift(24)
g["roll_24"] = shifted.rolling(24, min_periods=12).mean()
g["roll_168"] = shifted.rolling(168, min_periods=84).mean()
out.append(g)
return pd.concat(out, ignore_index=True)
def _add_seasonal_lags(df: pd.DataFrame) -> pd.DataFrame:
"""Previous observed same weekday/hour values, strictly in the past."""
out = df.sort_values(["sensor", "weekday", "hour", "t"]).copy()
grouped = out.groupby(["sensor", "weekday", "hour"], observed=True)["y"]
out["lag_week"] = grouped.shift(1)
prior = pd.concat(
[grouped.shift(k) for k in (2, 3, 4)],
axis=1,
)
prior.columns = ["lag_2w", "lag_3w", "lag_4w"]
all_prior = pd.concat([out["lag_week"], prior], axis=1)
counts = all_prior.notna().sum(axis=1)
out["seasonal_mean_4w"] = all_prior.mean(axis=1, skipna=True).where(counts >= 2)
by_hour = out.groupby(["sensor", "hour"], observed=True)["y"]
out["lag_day"] = by_hour.shift(1)
out["lag_2d"] = by_hour.shift(2)
return out.sort_values(["sensor", "t"]).reset_index(drop=True)
def build_features(
hourly: pd.DataFrame,
weather: pd.DataFrame,
*,
data_end: pd.Timestamp | None = None,
) -> pd.DataFrame:
"""Build the full feature table (observed hours only, target present)."""
hourly = hourly[hourly["v"].notna()].copy()
end = data_end or hourly["t"].max()
grid = _add_rollups(_grid(hourly, end))
df = hourly.merge(
grid[["sensor", "t", "y", "roll_24", "roll_168"]],
on=["sensor", "t"],
how="left",
)
df = _calendar(df)
df["sensor_code"] = df["sensor"].map(SENSOR_CODES).astype("int16")
df = _add_seasonal_lags(df)
df = df.merge(weather.drop(columns=[c for c in ("time",) if c in weather.columns]).assign(
t=weather["time"]
), on="t", how="left")
df = df[df["y"].notna()].copy()
return df.sort_values(["sensor", "t"]).reset_index(drop=True)
def feature_columns(kind: str) -> list[str]:
mapping = {
"day_no_holiday": config.FEATURE_DAY_NO_HOLIDAY,
"day": config.FEATURE_DAY,
"day_weather": config.FEATURE_DAY_WEATHER,
"week": config.FEATURE_WEEK,
"week_weather": config.FEATURE_WEEK_WEATHER,
}
try:
return list(mapping[kind])
except KeyError as exc:
raise ValueError(f"unknown feature set {kind!r}") from exc
def seasonal_lookup(hourly: pd.DataFrame) -> dict[str, dict[str, float]]:
"""Last-4-observation lookup per sensor/weekday/hour for future inference.
Key format: "<sensor>|<weekday>|<hour>" -> {"mean4": float, "last": float}.
This is the same definition used by ``lag_week`` / ``seasonal_mean_4w`` in
training, evaluated at the last observed week, so a future timestamp can use
it without assuming data after the CSV end.
"""
df = _calendar(hourly[hourly["v"].notna()].copy())
lookup: dict[str, dict[str, float]] = {}
for (sensor, weekday, hour), group in df.groupby(
["sensor", "weekday", "hour"], observed=True
):
last4 = group.sort_values("t")["v"].tail(4)
lookup[f"{sensor}|{int(weekday)}|{int(hour)}"] = {
"mean4": round(float(last4.mean()), 4),
"last": round(float(last4.iloc[-1]), 4),
}
return lookup
def future_calendar(start: pd.Timestamp, periods: int) -> pd.DataFrame:
"""Calendar features for a future hourly grid (no target, no lags)."""
t = pd.date_range(start, periods=periods, freq="h", tz="UTC")
frame = pd.DataFrame({"t": t})
frame = _calendar(frame)
return frame
+162
View File
@@ -0,0 +1,162 @@
"""Parse and clean the raw PAX CSV.
Verifies the claims in mission/data.md: 192,464 rows, 12 device IDs, 8 usable
logical locations, three excluded test devices, and the density-01 merge.
The raw file is never written to git; a cached parquet copy lives in data/cache/.
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
import pandas as pd
from . import config
log = logging.getLogger(__name__)
EXPECTED_CHECKS = {
"raw_rows": 192_464,
"density_rows": 96_232,
"battery_rows": 96_232,
"device_ids": 12,
"sensor_names": 11,
"usable_sensors": 8,
"density_01_rows_multi_sensor": 12_670,
}
def _logical_sensor(name: str, device_id: str) -> str:
return config.DEVICE_TO_SENSOR.get(device_id, name)
def load_raw(path: Path | None = None) -> pd.DataFrame:
path = path or config.raw_csv_path()
if not path.exists():
raise FileNotFoundError(
f"raw CSV not found at {path}. Set PULSECAST_RAW_CSV or place the "
"challenge download at that path (never commit it)."
)
df = pd.read_csv(path, parse_dates=["time"], dtype={"sensor_id": "string", "name": "string"})
df["time"] = pd.to_datetime(df["time"], utc=True)
expected = {"time", "sensor_id", "name", "longitude", "latitude", "type", "value"}
missing = expected - set(df.columns)
if missing:
raise ValueError(f"raw CSV missing columns: {sorted(missing)}")
return df
def clean(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
"""Return (density, battery, audit) with logical sensors and exclusions applied."""
density = df[df["type"] == "densityData"].copy()
battery = df[df["type"] == "batteryData"].copy()
for frame in (density, battery):
frame["sensor"] = [
_logical_sensor(n, d) for n, d in zip(frame["name"], frame["sensor_id"], strict=True)
]
excluded_rows = {
name: int((density["sensor"] == name).sum())
for name in config.EXCLUDED_SENSORS
}
density = density[~density["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
battery = battery[~battery["sensor"].isin(config.EXCLUDED_SENSORS)].copy()
before = len(density)
density = density[
density["value"].between(config.VALUE_MIN, config.VALUE_MAX, inclusive="both")
].copy()
dropped_out_of_range = before - len(density)
usable = sorted(density["sensor"].unique().tolist())
if set(usable) != set(config.USABLE_SENSORS):
raise ValueError(
f"usable sensor set changed: {usable}; expected {sorted(config.USABLE_SENSORS)}"
)
multi = density[density["time"] >= pd.Timestamp(config.MULTI_SENSOR_START)]
density_01_devices = (
multi[multi["sensor"] == "density-01"]
.groupby("sensor_id", observed=True)
.agg(rows=("value", "size"), first=("time", "min"), last=("time", "max"))
)
# One reading per (time, device, type); duplicates would double-weight a bin.
dup_keys = int(density.duplicated(subset=["time", "sensor_id"]).sum())
if dup_keys:
density = density.drop_duplicates(subset=["time", "sensor_id"], keep="last")
cadence = (
density.sort_values(["sensor", "time"])
.groupby("sensor", observed=True)["time"]
.diff()
.dt.total_seconds()
.div(60)
.dropna()
)
audit = {
"raw_rows": int(len(df)),
"density_rows": int(len(df[df["type"] == "densityData"])),
"battery_rows": int(len(df[df["type"] == "batteryData"])),
"device_ids": int(df["sensor_id"].nunique()),
"sensor_names": int(df["name"].nunique()),
"usable_sensors": len(usable),
"sensor_ids": usable,
"excluded_sensors": {
name: {"rows": excluded_rows[name], "reason": reason}
for name, reason in config.EXCLUDED_SENSORS.items()
},
"density_01_devices": {
str(device): {
"rows": int(row["rows"]),
"first": _iso(row["first"]),
"last": _iso(row["last"]),
}
for device, row in density_01_devices.iterrows()
},
"density_01_rows_multi_sensor": int(
(multi["sensor"] == "density-01").sum()
),
"time_range": {
"from": _iso(density["time"].min()),
"to": _iso(density["time"].max()),
},
"multi_sensor_from": _iso(multi["time"].min()),
"cadence_median_min": round(float(cadence.median()), 2),
"dropped_out_of_range": int(dropped_out_of_range),
"duplicate_keys_dropped": dup_keys,
"checks": [],
"all_checks_ok": True,
}
for key, expected in EXPECTED_CHECKS.items():
actual = audit[key]
ok = int(actual) == int(expected)
audit["checks"].append({"name": key, "expected": expected, "actual": actual, "ok": ok})
if not ok:
audit["all_checks_ok"] = False
log.warning("data audit mismatch %s: expected %s, got %s", key, expected, actual)
return density, battery, audit
def write_audit(audit: dict, path: Path | None = None) -> Path:
path = path or config.artifacts_dir() / "data_audit.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(audit, indent=2, ensure_ascii=False) + "\n")
return path
def load_cached_density() -> pd.DataFrame:
path = config.cache_dir() / "density_clean.parquet"
if not path.exists():
raise FileNotFoundError(f"{path} missing; run `make data` first")
return pd.read_parquet(path)
def _iso(ts) -> str | None:
if ts is None or pd.isna(ts):
return None
return pd.Timestamp(ts).tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ")
+190
View File
@@ -0,0 +1,190 @@
"""Open-Meteo weather: historical archive for training, forecast for scenarios.
No API key. Responses are cached under data/cache/ so `make train` and the
Docker image need no network. The raw CSV and cache stay out of git; the final
context.json is committed.
"""
from __future__ import annotations
import json
import logging
import time
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
import httpx
import pandas as pd
from . import config
log = logging.getLogger(__name__)
ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive"
FORECAST_URL = "https://api.open-meteo.com/v1/forecast"
ARCHIVE_FILE = "weather_archive.parquet"
FORECAST_FILE = "weather_forecast.parquet"
META_FILE = "weather_meta.json"
HOURLY_PARAMS = ",".join(config.WEATHER_FIELDS)
CURRENT_PARAMS = HOURLY_PARAMS
def _get_json(url: str, params: dict, *, retries: int = 3) -> dict:
last: Exception | None = None
for attempt in range(retries):
try:
with httpx.Client(timeout=30.0) as client:
resp = client.get(url, params=params)
resp.raise_for_status()
payload = resp.json()
if payload.get("error"):
raise RuntimeError(f"Open-Meteo error: {payload.get('reason', payload)}")
return payload
except Exception as exc: # network / HTTP / payload
last = exc
if attempt < retries - 1:
time.sleep(2**attempt)
raise RuntimeError(f"Open-Meteo request failed after {retries} attempts: {last}")
def _hourly_frame(payload: dict) -> pd.DataFrame:
hourly = payload.get("hourly") or {}
if "time" not in hourly:
raise RuntimeError("Open-Meteo payload has no hourly.time")
frame = pd.DataFrame(hourly)
frame["time"] = pd.to_datetime(frame["time"], utc=True)
missing = [f for f in config.WEATHER_FIELDS if f not in frame.columns]
if missing:
raise RuntimeError(f"Open-Meteo payload missing fields: {missing}")
frame = frame.rename(columns=config.WEATHER_COLUMNS)
cols = ["time"] + list(config.WEATHER_COLUMNS.values())
return frame[cols]
def _meta_path() -> Path:
return config.cache_dir() / META_FILE
def _load_meta() -> dict:
path = _meta_path()
if path.exists():
return json.loads(path.read_text())
return {}
def fetch_archive(start: date, end: date) -> pd.DataFrame:
payload = _get_json(
ARCHIVE_URL,
{
"latitude": config.LATITUDE,
"longitude": config.LONGITUDE,
"start_date": start.isoformat(),
"end_date": end.isoformat(),
"hourly": HOURLY_PARAMS,
"timezone": "UTC",
},
)
frame = _hourly_frame(payload)
log.info("fetched %s archive weather rows for %s..%s", len(frame), start, end)
return frame
def fetch_forecast(days: int = 8) -> tuple[pd.DataFrame, dict, dict]:
payload = _get_json(
FORECAST_URL,
{
"latitude": config.LATITUDE,
"longitude": config.LONGITUDE,
"hourly": HOURLY_PARAMS,
"current": CURRENT_PARAMS,
"forecast_days": days,
"timezone": "UTC",
},
)
frame = _hourly_frame(payload)
current = payload.get("current") or {}
now = {
"t": current.get("time"),
"temp_c": current.get("temperature_2m"),
"precip_mm": current.get("precipitation"),
"cloud_pct": current.get("cloud_cover"),
"wind_kmh": current.get("wind_speed_10m"),
"humidity_pct": current.get("relative_humidity_2m"),
}
grid = {"latitude": payload.get("latitude"), "longitude": payload.get("longitude")}
meta = {
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"current": now,
"grid": grid,
"source": "open-meteo",
}
log.info("fetched %s forecast weather rows", len(frame))
return frame, meta, grid
def ensure_weather(
data_start: pd.Timestamp | None = None,
data_end: pd.Timestamp | None = None,
*,
refresh: bool = False,
) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
"""Load cached weather, fetching only when missing, stale, or refresh=True."""
cache = config.cache_dir()
cache.mkdir(parents=True, exist_ok=True)
archive_path = cache / ARCHIVE_FILE
forecast_path = cache / FORECAST_FILE
meta = _load_meta()
if archive_path.exists() and not refresh:
archive = pd.read_parquet(archive_path)
else:
start = (data_start or pd.Timestamp(config.MULTI_SENSOR_START)).date() - timedelta(days=2)
end = (data_end or pd.Timestamp.now(tz="UTC")).date() + timedelta(days=1)
archive = fetch_archive(start, end)
archive.to_parquet(archive_path, index=False)
if forecast_path.exists() and not refresh:
forecast = pd.read_parquet(forecast_path)
else:
try:
forecast, meta, _ = fetch_forecast()
forecast.to_parquet(forecast_path, index=False)
_meta_path().write_text(json.dumps(meta, indent=2) + "\n")
except Exception:
if not forecast_path.exists():
raise
log.warning("Open-Meteo forecast refresh failed; using cached forecast")
forecast = pd.read_parquet(forecast_path)
if not meta:
meta = {
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"current": None,
"grid": {"latitude": config.LATITUDE, "longitude": config.LONGITUDE},
"source": "cache",
}
# Keep the archive within the challenge window (+/- 2 days) to avoid
# accidentally training on future weather if the cache is refreshed later.
if data_end is not None:
archive = archive[archive["time"] <= data_end + pd.Timedelta(days=1)].copy()
if data_start is not None:
archive = archive[archive["time"] >= data_start - pd.Timedelta(days=2)].copy()
return archive.reset_index(drop=True), forecast.reset_index(drop=True), meta
def load_cached_weather() -> tuple[pd.DataFrame, pd.DataFrame, dict]:
cache = config.cache_dir()
archive = pd.read_parquet(cache / ARCHIVE_FILE)
forecast = pd.read_parquet(cache / FORECAST_FILE)
return archive, forecast, _load_meta()
def refresh_weather() -> None:
archive, forecast, meta = ensure_weather(refresh=True)
log.info("weather refreshed: archive=%s forecast=%s", len(archive), len(forecast))
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
refresh_weather()
Generated
+1133
View File
File diff suppressed because it is too large. Load diff