chore: bootstrap ML-py scaffold
This commit is contained in:
commit
2eeb1f937a
12 files changed
+1924
No files matched your search
@@ -0,0 +1,190 @@
|
||||
"""Open-Meteo weather: historical archive for training, forecast for scenarios.
|
||||
|
||||
No API key. Responses are cached under data/cache/ so `make train` and the
|
||||
Docker image need no network. The raw CSV and cache stay out of git; the final
|
||||
context.json is committed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pandas as pd
|
||||
|
||||
from . import config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ARCHIVE_URL = "https://archive-api.open-meteo.com/v1/archive"
|
||||
FORECAST_URL = "https://api.open-meteo.com/v1/forecast"
|
||||
ARCHIVE_FILE = "weather_archive.parquet"
|
||||
FORECAST_FILE = "weather_forecast.parquet"
|
||||
META_FILE = "weather_meta.json"
|
||||
HOURLY_PARAMS = ",".join(config.WEATHER_FIELDS)
|
||||
CURRENT_PARAMS = HOURLY_PARAMS
|
||||
|
||||
|
||||
def _get_json(url: str, params: dict, *, retries: int = 3) -> dict:
|
||||
last: Exception | None = None
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
resp = client.get(url, params=params)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
if payload.get("error"):
|
||||
raise RuntimeError(f"Open-Meteo error: {payload.get('reason', payload)}")
|
||||
return payload
|
||||
except Exception as exc: # network / HTTP / payload
|
||||
last = exc
|
||||
if attempt < retries - 1:
|
||||
time.sleep(2**attempt)
|
||||
raise RuntimeError(f"Open-Meteo request failed after {retries} attempts: {last}")
|
||||
|
||||
|
||||
def _hourly_frame(payload: dict) -> pd.DataFrame:
|
||||
hourly = payload.get("hourly") or {}
|
||||
if "time" not in hourly:
|
||||
raise RuntimeError("Open-Meteo payload has no hourly.time")
|
||||
frame = pd.DataFrame(hourly)
|
||||
frame["time"] = pd.to_datetime(frame["time"], utc=True)
|
||||
missing = [f for f in config.WEATHER_FIELDS if f not in frame.columns]
|
||||
if missing:
|
||||
raise RuntimeError(f"Open-Meteo payload missing fields: {missing}")
|
||||
frame = frame.rename(columns=config.WEATHER_COLUMNS)
|
||||
cols = ["time"] + list(config.WEATHER_COLUMNS.values())
|
||||
return frame[cols]
|
||||
|
||||
|
||||
def _meta_path() -> Path:
|
||||
return config.cache_dir() / META_FILE
|
||||
|
||||
|
||||
def _load_meta() -> dict:
|
||||
path = _meta_path()
|
||||
if path.exists():
|
||||
return json.loads(path.read_text())
|
||||
return {}
|
||||
|
||||
|
||||
def fetch_archive(start: date, end: date) -> pd.DataFrame:
|
||||
payload = _get_json(
|
||||
ARCHIVE_URL,
|
||||
{
|
||||
"latitude": config.LATITUDE,
|
||||
"longitude": config.LONGITUDE,
|
||||
"start_date": start.isoformat(),
|
||||
"end_date": end.isoformat(),
|
||||
"hourly": HOURLY_PARAMS,
|
||||
"timezone": "UTC",
|
||||
},
|
||||
)
|
||||
frame = _hourly_frame(payload)
|
||||
log.info("fetched %s archive weather rows for %s..%s", len(frame), start, end)
|
||||
return frame
|
||||
|
||||
|
||||
def fetch_forecast(days: int = 8) -> tuple[pd.DataFrame, dict, dict]:
|
||||
payload = _get_json(
|
||||
FORECAST_URL,
|
||||
{
|
||||
"latitude": config.LATITUDE,
|
||||
"longitude": config.LONGITUDE,
|
||||
"hourly": HOURLY_PARAMS,
|
||||
"current": CURRENT_PARAMS,
|
||||
"forecast_days": days,
|
||||
"timezone": "UTC",
|
||||
},
|
||||
)
|
||||
frame = _hourly_frame(payload)
|
||||
current = payload.get("current") or {}
|
||||
now = {
|
||||
"t": current.get("time"),
|
||||
"temp_c": current.get("temperature_2m"),
|
||||
"precip_mm": current.get("precipitation"),
|
||||
"cloud_pct": current.get("cloud_cover"),
|
||||
"wind_kmh": current.get("wind_speed_10m"),
|
||||
"humidity_pct": current.get("relative_humidity_2m"),
|
||||
}
|
||||
grid = {"latitude": payload.get("latitude"), "longitude": payload.get("longitude")}
|
||||
meta = {
|
||||
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"current": now,
|
||||
"grid": grid,
|
||||
"source": "open-meteo",
|
||||
}
|
||||
log.info("fetched %s forecast weather rows", len(frame))
|
||||
return frame, meta, grid
|
||||
|
||||
|
||||
def ensure_weather(
|
||||
data_start: pd.Timestamp | None = None,
|
||||
data_end: pd.Timestamp | None = None,
|
||||
*,
|
||||
refresh: bool = False,
|
||||
) -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
||||
"""Load cached weather, fetching only when missing, stale, or refresh=True."""
|
||||
cache = config.cache_dir()
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
archive_path = cache / ARCHIVE_FILE
|
||||
forecast_path = cache / FORECAST_FILE
|
||||
meta = _load_meta()
|
||||
|
||||
if archive_path.exists() and not refresh:
|
||||
archive = pd.read_parquet(archive_path)
|
||||
else:
|
||||
start = (data_start or pd.Timestamp(config.MULTI_SENSOR_START)).date() - timedelta(days=2)
|
||||
end = (data_end or pd.Timestamp.now(tz="UTC")).date() + timedelta(days=1)
|
||||
archive = fetch_archive(start, end)
|
||||
archive.to_parquet(archive_path, index=False)
|
||||
|
||||
if forecast_path.exists() and not refresh:
|
||||
forecast = pd.read_parquet(forecast_path)
|
||||
else:
|
||||
try:
|
||||
forecast, meta, _ = fetch_forecast()
|
||||
forecast.to_parquet(forecast_path, index=False)
|
||||
_meta_path().write_text(json.dumps(meta, indent=2) + "\n")
|
||||
except Exception:
|
||||
if not forecast_path.exists():
|
||||
raise
|
||||
log.warning("Open-Meteo forecast refresh failed; using cached forecast")
|
||||
forecast = pd.read_parquet(forecast_path)
|
||||
|
||||
if not meta:
|
||||
meta = {
|
||||
"fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"current": None,
|
||||
"grid": {"latitude": config.LATITUDE, "longitude": config.LONGITUDE},
|
||||
"source": "cache",
|
||||
}
|
||||
|
||||
# Keep the archive within the challenge window (+/- 2 days) to avoid
|
||||
# accidentally training on future weather if the cache is refreshed later.
|
||||
if data_end is not None:
|
||||
archive = archive[archive["time"] <= data_end + pd.Timedelta(days=1)].copy()
|
||||
if data_start is not None:
|
||||
archive = archive[archive["time"] >= data_start - pd.Timedelta(days=2)].copy()
|
||||
return archive.reset_index(drop=True), forecast.reset_index(drop=True), meta
|
||||
|
||||
|
||||
def load_cached_weather() -> tuple[pd.DataFrame, pd.DataFrame, dict]:
|
||||
cache = config.cache_dir()
|
||||
archive = pd.read_parquet(cache / ARCHIVE_FILE)
|
||||
forecast = pd.read_parquet(cache / FORECAST_FILE)
|
||||
return archive, forecast, _load_meta()
|
||||
|
||||
|
||||
def refresh_weather() -> None:
|
||||
archive, forecast, meta = ensure_weather(refresh=True)
|
||||
log.info("weather refreshed: archive=%s forecast=%s", len(archive), len(forecast))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
refresh_weather()
|
||||
Reference in new issue
Block a user