- finish the global HistGB bundle, scenario forecasts and 28-day ablation - export sensors/series/forecasts/metrics/context plus baked weather frames - add FastAPI health/meta/predict/simulate and background /train jobs - add the p75 busy classifier, retraining from CSV, and hot-swap persistence - add pytest coverage, multi-stage non-root Dockerfile, README and model card Holdout: weather 7d MAE 3.32 / WAPE 27.5% / R2 0.777 vs seasonal naive MAE 4.23 / WAPE 35.0% / R2 0.656 (5,350 eval rows).
191 lines
6.5 KiB
Python
191 lines
6.5 KiB
Python
"""Open-Meteo weather: historical archive for training, forecast for scenarios.
|
|
|
|
No API key. Responses are cached under data/cache/ so `make train` and the
|
|
Docker image need no network. The raw CSV and cache stay out of git; the final
|
|
context.json is committed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import time
|
|
from datetime import UTC, date, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pandas as pd
|
|
|
|
from . import config
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive"
|
|
FORECAST_URL = "https://api.open-meteo.com/v1/forecast"
|
|
ARCHIVE_FILE = "weather_archive.parquet"
|
|
FORECAST_FILE = "weather_forecast.parquet"
|
|
META_FILE = "weather_meta.json"
|
|
HOURLY_PARAMS = ",".join(config.WEATHER_FIELDS)
|
|
CURRENT_PARAMS = HOURLY_PARAMS
|
|
|
|
|
|
def _get_json(url: str, params: dict, *, retries: int = 3) -> dict:
|
|
last: Exception | None = None
|
|
for attempt in range(retries):
|
|
try:
|
|
with httpx.Client(timeout=30.0) as client:
|
|
resp = client.get(url, params=params)
|
|
resp.raise_for_status()
|
|
payload = resp.json()
|
|
if payload.get("error"):
|
|
raise RuntimeError(f"Open-Meteo error: {payload.get('reason', payload)}")
|
|
return payload
|
|
except Exception as exc: # noqa: BLE001 - network/HTTP/payload retry boundary
|
|
last = exc
|
|
if attempt < retries - 1:
|
|
time.sleep(2**attempt)
|
|
raise RuntimeError(f"Open-Meteo request failed after {retries} attempts: {last}")
|
|
|
|
|
|
def _hourly_frame(payload: dict) -> pd.DataFrame:
|
|
hourly = payload.get("hourly") or {}
|
|
if "time" not in hourly:
|
|
raise RuntimeError("Open-Meteo payload has no hourly.time")
|
|
frame = pd.DataFrame(hourly)
|
|
frame["time"] = pd.to_datetime(frame["time"], utc=True)
|
|
missing = [f for f in config.WEATHER_FIELDS if f not in frame.columns]
|
|
if missing:
|
|
raise RuntimeError(f"Open-Meteo payload missing fields: {missing}")
|
|
frame = frame.rename(columns=config.WEATHER_COLUMNS)
|
|
cols = ["time"] + list(config.WEATHER_COLUMNS.values())
|
|
return frame[cols]
|
|
|
|
|
|
def _meta_path() -> Path:
|
|
return config.cache_dir() / META_FILE
|
|
|
|
|
|
def _load_meta() -> dict:
|
|
path = _meta_path()
|
|
if path.exists():
|
|
return json.loads(path.read_text())
|
|
return {}
|
|
|
|
|
|
def fetch_archive(start: date, end: date) -> pd.DataFrame:
|
|
payload = _get_json(
|
|
ARCHIVE_URL,
|
|
{
|
|
"latitude": config.LATITUDE,
|
|
"longitude": config.LONGITUDE,
|
|
"start_date": start.isoformat(),
|
|
"end_date": end.isoformat(),
|
|
"hourly": HOURLY_PARAMS,
|
|
"timezone": "UTC",
|
|
},
|
|
)
|
|
frame = _hourly_frame(payload)
|
|
log.info("fetched %s archive weather rows for %s..%s", len(frame), start, end)
|
|
return frame
|
|
|
|
|
|
def fetch_forecast(days: int = 8) -> tuple[pd.DataFrame, dict, dict]:
|
|
payload = _get_json(
|
|
FORECAST_URL,
|
|
{
|
|
"latitude": config.LATITUDE,
|
|
"longitude": config.LONGITUDE,
|
|
"hourly": HOURLY_PARAMS,
|
|
"current": CURRENT_PARAMS,
|
|
"forecast_days": days,
|
|
"timezone": "UTC",
|
|
},
|
|
)
|
|
frame = _hourly_frame(payload)
|
|
current = payload.get("current") or {}
|
|
now = {
|
|
"t": current.get("time"),
|
|
"temp_c": current.get("temperature_2m"),
|
|
"precip_mm": current.get("precipitation"),
|
|
"cloud_pct": current.get("cloud_cover"),
|
|
"wind_kmh": current.get("wind_speed_10m"),
|
|
"humidity_pct": current.get("relative_humidity_2m"),
|
|
}
|
|
grid = {"latitude": payload.get("latitude"), "longitude": payload.get("longitude")}
|
|
meta = {
|
|
"fetched_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"current": now,
|
|
"grid": grid,
|
|
"source": "open-meteo",
|
|
}
|
|
log.info("fetched %s forecast weather rows", len(frame))
|
|
return frame, meta, grid
|
|
|
|
|
|
def ensure_weather(
|
|
data_start: pd.Timestamp | None = None,
|
|
data_end: pd.Timestamp | None = None,
|
|
*,
|
|
refresh: bool = False,
|
|
) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
|
"""Load cached weather, fetching only when missing, stale, or refresh=True."""
|
|
cache = config.cache_dir()
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
archive_path = cache / ARCHIVE_FILE
|
|
forecast_path = cache / FORECAST_FILE
|
|
meta = _load_meta()
|
|
|
|
if archive_path.exists() and not refresh:
|
|
archive = pd.read_parquet(archive_path)
|
|
else:
|
|
start = (data_start or pd.Timestamp(config.MULTI_SENSOR_START)).date() - timedelta(days=2)
|
|
end = (data_end or pd.Timestamp.now(tz="UTC")).date() + timedelta(days=1)
|
|
archive = fetch_archive(start, end)
|
|
archive.to_parquet(archive_path, index=False)
|
|
|
|
if forecast_path.exists() and not refresh:
|
|
forecast = pd.read_parquet(forecast_path)
|
|
else:
|
|
try:
|
|
forecast, meta, _ = fetch_forecast()
|
|
forecast.to_parquet(forecast_path, index=False)
|
|
_meta_path().write_text(json.dumps(meta, indent=2) + "\n")
|
|
except Exception:
|
|
if not forecast_path.exists():
|
|
raise
|
|
log.warning("Open-Meteo forecast refresh failed; using cached forecast")
|
|
forecast = pd.read_parquet(forecast_path)
|
|
|
|
if not meta:
|
|
meta = {
|
|
"fetched_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"current": None,
|
|
"grid": {"latitude": config.LATITUDE, "longitude": config.LONGITUDE},
|
|
"source": "cache",
|
|
}
|
|
|
|
# Keep the archive within the challenge window (+/- 2 days) to avoid
|
|
# accidentally training on future weather if the cache is refreshed later.
|
|
if data_end is not None:
|
|
archive = archive[archive["time"] <= data_end + pd.Timedelta(days=1)].copy()
|
|
if data_start is not None:
|
|
archive = archive[archive["time"] >= data_start - pd.Timedelta(days=2)].copy()
|
|
return archive.reset_index(drop=True), forecast.reset_index(drop=True), meta
|
|
|
|
|
|
def load_cached_weather() -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
|
cache = config.cache_dir()
|
|
archive = pd.read_parquet(cache / ARCHIVE_FILE)
|
|
forecast = pd.read_parquet(cache / FORECAST_FILE)
|
|
return archive, forecast, _load_meta()
|
|
|
|
|
|
def refresh_weather() -> None:
|
|
archive, forecast, _ = ensure_weather(refresh=True)
|
|
log.info("weather refreshed: archive=%s forecast=%s", len(archive), len(forecast))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
|
refresh_weather()
|