"""Open-Meteo weather: historical archive for training, forecast for scenarios. No API key. Responses are cached under data/cache/ so `make train` and the Docker image need no network. The raw CSV and cache stay out of git; the final context.json is committed. """ from __future__ import annotations import json import logging import time from datetime import UTC, date, datetime, timedelta from pathlib import Path import httpx import pandas as pd from . import config log = logging.getLogger(__name__) ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive" FORECAST_URL = "https://api.open-meteo.com/v1/forecast" ARCHIVE_FILE = "weather_archive.parquet" FORECAST_FILE = "weather_forecast.parquet" META_FILE = "weather_meta.json" HOURLY_PARAMS = ",".join(config.WEATHER_FIELDS) CURRENT_PARAMS = HOURLY_PARAMS def _get_json(url: str, params: dict, *, retries: int = 3) -> dict: last: Exception | None = None for attempt in range(retries): try: with httpx.Client(timeout=30.0) as client: resp = client.get(url, params=params) resp.raise_for_status() payload = resp.json() if payload.get("error"): raise RuntimeError(f"Open-Meteo error: {payload.get('reason', payload)}") return payload except Exception as exc: # noqa: BLE001 - network/HTTP/payload retry boundary last = exc if attempt < retries - 1: time.sleep(2**attempt) raise RuntimeError(f"Open-Meteo request failed after {retries} attempts: {last}") def _hourly_frame(payload: dict) -> pd.DataFrame: hourly = payload.get("hourly") or {} if "time" not in hourly: raise RuntimeError("Open-Meteo payload has no hourly.time") frame = pd.DataFrame(hourly) frame["time"] = pd.to_datetime(frame["time"], utc=True) missing = [f for f in config.WEATHER_FIELDS if f not in frame.columns] if missing: raise RuntimeError(f"Open-Meteo payload missing fields: {missing}") frame = frame.rename(columns=config.WEATHER_COLUMNS) cols = ["time"] + list(config.WEATHER_COLUMNS.values()) return frame[cols] def _meta_path() -> Path: return config.cache_dir() / META_FILE def _load_meta() -> dict: path = _meta_path() if path.exists(): return json.loads(path.read_text()) return {} def fetch_archive(start: date, end: date) -> pd.DataFrame: payload = _get_json( ARCHIVE_URL, { "latitude": config.LATITUDE, "longitude": config.LONGITUDE, "start_date": start.isoformat(), "end_date": end.isoformat(), "hourly": HOURLY_PARAMS, "timezone": "UTC", }, ) frame = _hourly_frame(payload) log.info("fetched %s archive weather rows for %s..%s", len(frame), start, end) return frame def fetch_forecast(days: int = 8) -> tuple[pd.DataFrame, dict, dict]: payload = _get_json( FORECAST_URL, { "latitude": config.LATITUDE, "longitude": config.LONGITUDE, "hourly": HOURLY_PARAMS, "current": CURRENT_PARAMS, "forecast_days": days, "timezone": "UTC", }, ) frame = _hourly_frame(payload) current = payload.get("current") or {} now = { "t": current.get("time"), "temp_c": current.get("temperature_2m"), "precip_mm": current.get("precipitation"), "cloud_pct": current.get("cloud_cover"), "wind_kmh": current.get("wind_speed_10m"), "humidity_pct": current.get("relative_humidity_2m"), } grid = {"latitude": payload.get("latitude"), "longitude": payload.get("longitude")} meta = { "fetched_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "current": now, "grid": grid, "source": "open-meteo", } log.info("fetched %s forecast weather rows", len(frame)) return frame, meta, grid def ensure_weather( data_start: pd.Timestamp | None = None, data_end: pd.Timestamp | None = None, *, refresh: bool = False, ) -> tuple[pd.DataFrame, pd.DataFrame, dict]: """Load cached weather, fetching only when missing, stale, or refresh=True.""" cache = config.cache_dir() cache.mkdir(parents=True, exist_ok=True) archive_path = cache / ARCHIVE_FILE forecast_path = cache / FORECAST_FILE meta = _load_meta() if archive_path.exists() and not refresh: archive = pd.read_parquet(archive_path) else: start = (data_start or pd.Timestamp(config.MULTI_SENSOR_START)).date() - timedelta(days=2) end = (data_end or pd.Timestamp.now(tz="UTC")).date() + timedelta(days=1) archive = fetch_archive(start, end) archive.to_parquet(archive_path, index=False) if forecast_path.exists() and not refresh: forecast = pd.read_parquet(forecast_path) else: try: forecast, meta, _ = fetch_forecast() forecast.to_parquet(forecast_path, index=False) _meta_path().write_text(json.dumps(meta, indent=2) + "\n") except Exception: if not forecast_path.exists(): raise log.warning("Open-Meteo forecast refresh failed; using cached forecast") forecast = pd.read_parquet(forecast_path) if not meta: meta = { "fetched_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "current": None, "grid": {"latitude": config.LATITUDE, "longitude": config.LONGITUDE}, "source": "cache", } # Keep the archive within the challenge window (+/- 2 days) to avoid # accidentally training on future weather if the cache is refreshed later. if data_end is not None: archive = archive[archive["time"] <= data_end + pd.Timedelta(days=1)].copy() if data_start is not None: archive = archive[archive["time"] >= data_start - pd.Timedelta(days=2)].copy() return archive.reset_index(drop=True), forecast.reset_index(drop=True), meta def load_cached_weather() -> tuple[pd.DataFrame, pd.DataFrame, dict]: cache = config.cache_dir() archive = pd.read_parquet(cache / ARCHIVE_FILE) forecast = pd.read_parquet(cache / FORECAST_FILE) return archive, forecast, _load_meta() def refresh_weather() -> None: archive, forecast, _ = ensure_weather(refresh=True) log.info("weather refreshed: archive=%s forecast=%s", len(archive), len(forecast)) if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") refresh_weather()