chore: bootstrap ML-py scaffold

This commit is contained in:
Noel Sabia committed 2026-11-14 04:16:22 +01:00
commit 2eeb1f937a
12 files changed
+1924

No files matched your search

+190
View File
@@ -0,0 +1,190 @@
"""Open-Meteo weather: historical archive for training, forecast for scenarios.
No API key. Responses are cached under data/cache/ so `make train` and the
Docker image need no network. The raw CSV and cache stay out of git; the final
context.json is committed.
"""
from __future__ import annotations
import json
import logging
import time
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
import httpx
import pandas as pd
from . import config
log = logging.getLogger(__name__)
ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive"
FORECAST_URL = "https://api.open-meteo.com/v1/forecast"
ARCHIVE_FILE = "weather_archive.parquet"
FORECAST_FILE = "weather_forecast.parquet"
META_FILE = "weather_meta.json"
HOURLY_PARAMS = ",".join(config.WEATHER_FIELDS)
CURRENT_PARAMS = HOURLY_PARAMS
def _get_json(url: str, params: dict, *, retries: int = 3) -> dict:
last: Exception | None = None
for attempt in range(retries):
try:
with httpx.Client(timeout=30.0) as client:
resp = client.get(url, params=params)
resp.raise_for_status()
payload = resp.json()
if payload.get("error"):
raise RuntimeError(f"Open-Meteo error: {payload.get('reason', payload)}")
return payload
except Exception as exc: # network / HTTP / payload
last = exc
if attempt < retries - 1:
time.sleep(2**attempt)
raise RuntimeError(f"Open-Meteo request failed after {retries} attempts: {last}")
def _hourly_frame(payload: dict) -> pd.DataFrame:
hourly = payload.get("hourly") or {}
if "time" not in hourly:
raise RuntimeError("Open-Meteo payload has no hourly.time")
frame = pd.DataFrame(hourly)
frame["time"] = pd.to_datetime(frame["time"], utc=True)
missing = [f for f in config.WEATHER_FIELDS if f not in frame.columns]
if missing:
raise RuntimeError(f"Open-Meteo payload missing fields: {missing}")
frame = frame.rename(columns=config.WEATHER_COLUMNS)
cols = ["time"] + list(config.WEATHER_COLUMNS.values())
return frame[cols]
def _meta_path() -> Path:
return config.cache_dir() / META_FILE
def _load_meta() -> dict:
path = _meta_path()
if path.exists():
return json.loads(path.read_text())
return {}
def fetch_archive(start: date, end: date) -> pd.DataFrame:
payload = _get_json(
ARCHIVE_URL,
{
"latitude": config.LATITUDE,
"longitude": config.LONGITUDE,
"start_date": start.isoformat(),
"end_date": end.isoformat(),
"hourly": HOURLY_PARAMS,
"timezone": "UTC",
},
)
frame = _hourly_frame(payload)
log.info("fetched %s archive weather rows for %s..%s", len(frame), start, end)
return frame
def fetch_forecast(days: int = 8) -> tuple[pd.DataFrame, dict, dict]:
payload = _get_json(
FORECAST_URL,
{
"latitude": config.LATITUDE,
"longitude": config.LONGITUDE,
"hourly": HOURLY_PARAMS,
"current": CURRENT_PARAMS,
"forecast_days": days,
"timezone": "UTC",
},
)
frame = _hourly_frame(payload)
current = payload.get("current") or {}
now = {
"t": current.get("time"),
"temp_c": current.get("temperature_2m"),
"precip_mm": current.get("precipitation"),
"cloud_pct": current.get("cloud_cover"),
"wind_kmh": current.get("wind_speed_10m"),
"humidity_pct": current.get("relative_humidity_2m"),
}
grid = {"latitude": payload.get("latitude"), "longitude": payload.get("longitude")}
meta = {
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"current": now,
"grid": grid,
"source": "open-meteo",
}
log.info("fetched %s forecast weather rows", len(frame))
return frame, meta, grid
def ensure_weather(
data_start: pd.Timestamp | None = None,
data_end: pd.Timestamp | None = None,
*,
refresh: bool = False,
) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
"""Load cached weather, fetching only when missing, stale, or refresh=True."""
cache = config.cache_dir()
cache.mkdir(parents=True, exist_ok=True)
archive_path = cache / ARCHIVE_FILE
forecast_path = cache / FORECAST_FILE
meta = _load_meta()
if archive_path.exists() and not refresh:
archive = pd.read_parquet(archive_path)
else:
start = (data_start or pd.Timestamp(config.MULTI_SENSOR_START)).date() - timedelta(days=2)
end = (data_end or pd.Timestamp.now(tz="UTC")).date() + timedelta(days=1)
archive = fetch_archive(start, end)
archive.to_parquet(archive_path, index=False)
if forecast_path.exists() and not refresh:
forecast = pd.read_parquet(forecast_path)
else:
try:
forecast, meta, _ = fetch_forecast()
forecast.to_parquet(forecast_path, index=False)
_meta_path().write_text(json.dumps(meta, indent=2) + "\n")
except Exception:
if not forecast_path.exists():
raise
log.warning("Open-Meteo forecast refresh failed; using cached forecast")
forecast = pd.read_parquet(forecast_path)
if not meta:
meta = {
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"current": None,
"grid": {"latitude": config.LATITUDE, "longitude": config.LONGITUDE},
"source": "cache",
}
# Keep the archive within the challenge window (+/- 2 days) to avoid
# accidentally training on future weather if the cache is refreshed later.
if data_end is not None:
archive = archive[archive["time"] <= data_end + pd.Timedelta(days=1)].copy()
if data_start is not None:
archive = archive[archive["time"] >= data_start - pd.Timedelta(days=2)].copy()
return archive.reset_index(drop=True), forecast.reset_index(drop=True), meta
def load_cached_weather() -> tuple[pd.DataFrame, pd.DataFrame, dict]:
cache = config.cache_dir()
archive = pd.read_parquet(cache / ARCHIVE_FILE)
forecast = pd.read_parquet(cache / FORECAST_FILE)
return archive, forecast, _load_meta()
def refresh_weather() -> None:
archive, forecast, meta = ensure_weather(refresh=True)
log.info("weather refreshed: archive=%s forecast=%s", len(archive), len(forecast))
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
refresh_weather()