Files
paper-how-to-use-this-techn…/research/scripts/01_data_quality.py
T

1197 lines
53 KiB
Python

#!/usr/bin/env python3
"""Data-quality audit of the Schwarz PAX BLE counter export.
Reads the raw CSV read-only, computes schema/coverage/cadence/correlation/
zero-inflation/battery/anomaly statistics, writes CSV+JSON tables and PNG
figures under paper/research/. No raw data is modified.
Run:
uv run --with pandas --with matplotlib python paper/research/scripts/01_data_quality.py
"""
from __future__ import annotations
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from zoneinfo import ZoneInfo
ROOT = Path("/home/noel/Documents/Hackathon_2026")
RAW = ROOT / "data" / "raw" / "schwarz-pax.csv"
OUT = ROOT / "paper" / "research"
TAB = OUT / "outputs"
FIG = OUT / "figures"
for d in (TAB, FIG, OUT / "scripts"):
d.mkdir(parents=True, exist_ok=True)
TZ = ZoneInfo("Europe/Berlin")
SLOT = "15min"
# Panel window: full days after all eight core locations are live.
PANEL_START = pd.Timestamp("2026-06-18", tz="UTC")
PANEL_END = pd.Timestamp("2026-11-08", tz="UTC") # exclusive
PANEL_DAYS = int((PANEL_END - PANEL_START).days)
PANEL_SLOTS_TOTAL = int((PANEL_END - PANEL_START) / pd.Timedelta(SLOT))
# Two device IDs carry the same location name (hardware replacement).
LOC_MAP = {"863f75b0": "density-01", "f8c19730": "density-01"}
DEVICE_LABEL = {
"863f75b0": "density-01 (old)",
"f8c19730": "density-01 (new)",
}
plt.rcParams.update(
{
"figure.dpi": 150,
"savefig.dpi": 180,
"font.size": 8,
"axes.titlesize": 9,
"axes.labelsize": 8,
"axes.grid": True,
"grid.alpha": 0.25,
"axes.spines.top": False,
"axes.spines.right": False,
}
)
def pct(x: float, nd: int = 1) -> float:
return round(float(x) * 100, nd)
def md_table(df: pd.DataFrame, floatfmt: str = "{:.2f}") -> str:
"""Small markdown table writer that does not need tabulate."""
df = df.copy()
cols = list(df.columns)
lines = ["| " + " | ".join(map(str, cols)) + " |",
"|" + "|".join(["---"] * len(cols)) + "|"]
for _, row in df.iterrows():
vals = []
for c in cols:
v = row[c]
if isinstance(v, (float, np.floating)):
vals.append("" if pd.isna(v) else floatfmt.format(v))
else:
vals.append(str(v))
lines.append("| " + " | ".join(vals) + " |")
return "\n".join(lines)
def main() -> None:
raw = pd.read_csv(RAW, parse_dates=["time"])
raw["time"] = pd.to_datetime(raw["time"], utc=True)
raw = raw.sort_values(["sensor_id", "time"]).reset_index(drop=True)
dens = raw[raw["type"] == "densityData"].copy()
batt = raw[raw["type"] == "batteryData"].copy()
# ---------------------------------------------------------------- schema
schema = pd.DataFrame(
{
"column": raw.columns,
"dtype": [str(t) for t in raw.dtypes],
"non_null": [int(raw[c].notna().sum()) for c in raw.columns],
"null": [int(raw[c].isna().sum()) for c in raw.columns],
"example": [raw[c].iloc[0] for c in raw.columns],
}
)
schema.to_csv(TAB / "schema.csv", index=False)
inventory = (
raw.groupby(["sensor_id", "name", "longitude", "latitude"], dropna=False)
.agg(rows=("value", "size"), first=("time", "min"), last=("time", "max"))
.reset_index()
)
inventory["readings_density"] = inventory["sensor_id"].map(
dens.groupby("sensor_id").size()
)
inventory["readings_battery"] = inventory["sensor_id"].map(
batt.groupby("sensor_id").size()
)
inventory["location"] = inventory["sensor_id"].map(LOC_MAP).fillna(inventory["name"])
inventory["role"] = np.where(
inventory["readings_density"] >= 1000, "core", "episodic"
)
inventory = inventory.sort_values(["role", "location", "first"]).reset_index(drop=True)
inventory.to_csv(TAB / "sensor_inventory.csv", index=False)
core_ids = inventory.loc[inventory.role == "core", "sensor_id"].tolist()
episodic_ids = inventory.loc[inventory.role == "episodic", "sensor_id"].tolist()
core = dens[dens.sensor_id.isin(core_ids)].copy()
core["slot"] = core["time"].dt.floor(SLOT)
core["local"] = core["time"].dt.tz_convert(TZ)
# -------------------------------------------------------------- cadence
cadence_rows = []
interval_rows = []
for sid, g in core.groupby("sensor_id"):
g = g.sort_values("time")
dt = g["time"].diff().dt.total_seconds().dropna()
dt = dt[dt > 0]
near = dt[dt < 3600]
cadence_rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"n": len(g),
"median_s": near.median(),
"p05_s": near.quantile(0.05),
"p95_s": near.quantile(0.95),
"median_min": near.median() / 60,
"p05_min": near.quantile(0.05) / 60,
"p25_min": near.quantile(0.25) / 60,
"p75_min": near.quantile(0.75) / 60,
"p95_min": near.quantile(0.95) / 60,
"share_14_17min": ((near >= 840) & (near <= 1020)).mean(),
"share_28_34min": ((near >= 1680) & (near <= 2040)).mean(),
"max_gap_h": dt.max() / 3600,
}
)
for v in dt:
interval_rows.append((sid, v / 60))
cadence = pd.DataFrame(cadence_rows)
cadence.to_csv(TAB / "cadence_by_sensor.csv", index=False)
intervals = pd.DataFrame(interval_rows, columns=["sensor_id", "minutes"])
bins = [0, 5, 10, 14, 17, 20, 28, 34, 40, 60, 360, 1440, np.inf]
labels = ["<5m", "5-10m", "10-14m", "14-17m", "17-20m", "20-28m",
"28-34m", "34-40m", "40-60m", "1-6h", "6-24h", ">24h"]
intervals["bin"] = pd.cut(intervals["minutes"], bins=bins, labels=labels, right=False)
cadence_bins = (
intervals.groupby(["sensor_id", "bin"], observed=False).size().unstack(fill_value=0)
)
cadence_bins.to_csv(TAB / "cadence_bins.csv")
cadence_bin_share = cadence_bins.div(cadence_bins.sum(axis=1), axis=0)
# ------------------------------------------------- coverage / basic stats
rows = []
for sid, g in core.groupby("sensor_id"):
g = g.sort_values("time")
n = len(g)
span_s = (g["time"].iloc[-1] - g["time"].iloc[0]).total_seconds()
first_slot = g["time"].iloc[0].floor(SLOT)
last_slot = g["time"].iloc[-1].floor(SLOT)
expected_active = int((last_slot - first_slot) / pd.Timedelta(SLOT)) + 1
observed_active = g["slot"].nunique()
panel = g[(g.slot >= PANEL_START) & (g.slot < PANEL_END)]
panel_slots_total = int((PANEL_END - PANEL_START) / pd.Timedelta(SLOT))
dt = g["time"].diff().dt.total_seconds().dropna()
dt = dt[dt > 0]
zero = g["value"] == 0
runs = (zero != zero.shift()).cumsum()
zero_runs = zero.groupby(runs).sum()
panel = g[(g.slot >= PANEL_START) & (g.slot < PANEL_END)]
panel_slots_total = int((PANEL_END - PANEL_START) / pd.Timedelta(SLOT))
panel_hours = (PANEL_END - PANEL_START).total_seconds() / 3600
panel_gaps = panel["time"].diff().dt.total_seconds().dropna() / 3600
panel_gaps = panel_gaps[panel_gaps > 0]
median_min = dt[dt < 3600].median() / 60
cadence_ceiling = 100 * 15 / median_min
rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"role": "core",
"n": n,
"first": g["time"].iloc[0].strftime("%Y-%m-%d %H:%M"),
"last": g["time"].iloc[-1].strftime("%Y-%m-%d %H:%M"),
"span_days": span_s / 86400,
"readings_per_day": n / (span_s / 86400),
"expected_slots_active": expected_active,
"observed_slots_active": observed_active,
"coverage_active_pct": 100 * observed_active / expected_active,
"panel_slots": len(panel),
"panel_readings_per_day": len(panel) / 143,
"coverage_panel_pct": 100 * len(panel) / panel_slots_total,
"median_cadence_min": median_min,
"cadence_ceiling_pct": cadence_ceiling,
"coverage_vs_ceiling_pct": 100 * (len(panel) / panel_slots_total) / (cadence_ceiling / 100),
"time_in_gaps_gt1h_pct": 100 * panel_gaps[panel_gaps > 1].sum() / panel_hours,
"time_in_gaps_gt6h_pct": 100 * panel_gaps[panel_gaps > 6].sum() / panel_hours,
"panel_max_gap_h": float(panel_gaps.max()) if len(panel_gaps) else np.nan,
"panel_gaps_gt_1h": int((panel_gaps > 1).sum()),
"panel_gaps_gt_1d": int((panel_gaps > 24).sum()),
"zero_frac": zero.mean(),
"lt1_frac": (g["value"] < 1).mean(),
"lt5_frac": (g["value"] < 5).mean(),
"mean": g["value"].mean(),
"median": g["value"].median(),
"p95": g["value"].quantile(0.95),
"p99": g["value"].quantile(0.99),
"max": g["value"].max(),
"min": g["value"].min(),
"max_gap_h": dt.max() / 3600,
"gaps_gt_1h": int((dt > 3600).sum()),
"gaps_gt_6h": int((dt > 6 * 3600).sum()),
"gaps_gt_1d": int((dt > 86400).sum()),
"gaps_gt_7d": int((dt > 7 * 86400).sum()),
"max_zero_run": int(zero_runs.max()) if len(zero_runs) else 0,
"zero_runs_ge5": int((zero_runs >= 5).sum()),
}
)
for sid in episodic_ids:
g = dens[dens.sensor_id == sid].sort_values("time")
dt = g["time"].diff().dt.total_seconds().dropna()
rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"role": "episodic",
"n": len(g),
"first": g["time"].iloc[0].strftime("%Y-%m-%d %H:%M"),
"last": g["time"].iloc[-1].strftime("%Y-%m-%d %H:%M"),
"span_days": (g.time.iloc[-1] - g.time.iloc[0]).total_seconds() / 86400,
"readings_per_day": np.nan,
"coverage_active_pct": np.nan,
"coverage_panel_pct": np.nan,
"zero_frac": (g.value == 0).mean(),
"lt1_frac": (g.value < 1).mean(),
"lt5_frac": (g.value < 5).mean(),
"mean": g.value.mean(),
"median": g.value.median(),
"p95": g.value.quantile(0.95),
"p99": g.value.quantile(0.99),
"max": g.value.max(),
"min": g.value.min(),
"max_gap_h": (dt.max() / 3600) if len(dt) else np.nan,
"gaps_gt_1h": int((dt > 3600).sum()) if len(dt) else 0,
"gaps_gt_6h": int((dt > 6 * 3600).sum()) if len(dt) else 0,
"gaps_gt_1d": int((dt > 86400).sum()) if len(dt) else 0,
"gaps_gt_7d": int((dt > 7 * 86400).sum()) if len(dt) else 0,
}
)
coverage = pd.DataFrame(rows)
coverage.to_csv(TAB / "coverage_metrics.csv", index=False)
# ----------------------------------------------------------- duplicates
dup = {
"rows_total": int(len(raw)),
"exact_duplicate_rows": int(raw.duplicated().sum()),
"duplicate_time_sensor_type": int(raw.duplicated(["time", "sensor_id", "type"]).sum()),
"duplicate_time_sensor": int(raw.duplicated(["time", "sensor_id"]).sum()),
"duplicate_sensor_name": int(inventory["name"].duplicated().sum()),
"density_negative": int((dens["value"] < 0).sum()),
"density_nan": int(dens["value"].isna().sum()),
"battery_negative": int((batt["value"] < 0).sum()),
"battery_over_100": int((batt["value"] > 100).sum()),
"battery_nan": int(batt["value"].isna().sum()),
"intervals_lt_60s_core": int(
(core.groupby("sensor_id")["time"].diff().dt.total_seconds() < 60).sum()
),
}
# -------------------------------------------------------- battery merge
bsmall = batt[["time", "sensor_id", "value"]].rename(
columns={"time": "battery_time", "value": "battery"}
).sort_values("battery_time")
dsmall = dens[["time", "sensor_id", "value", "name"]].sort_values("time")
paired = pd.merge_asof(
dsmall,
bsmall,
left_on="time",
right_on="battery_time",
by="sensor_id",
direction="nearest",
tolerance=pd.Timedelta("60s"),
)
paired["offset_s"] = (paired["battery_time"] - paired["time"]).dt.total_seconds()
paired["slot"] = paired["time"].dt.floor(SLOT)
paired["local"] = paired["time"].dt.tz_convert(TZ)
battery_rows = []
for sid, g in paired.groupby("sensor_id"):
g = g.sort_values("time")
if g["battery"].isna().all():
continue
same = (g["battery"].diff() == 0)
run_id = (same != same.shift()).cumsum()
runs = g.groupby(run_id).size()
battery_rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"n": len(g),
"battery_n": int(g["battery"].notna().sum()),
"battery_missing": int(g["battery"].isna().sum()),
"first": g["battery"].iloc[0],
"last": g["battery"].iloc[-1],
"min": g["battery"].min(),
"p05": g["battery"].quantile(0.05),
"median": g["battery"].median(),
"mean": g["battery"].mean(),
"max": g["battery"].max(),
"share_lt10": (g["battery"] < 10).mean(),
"share_lt20": (g["battery"] < 20).mean(),
"share_zero": (g["battery"] == 0).mean(),
"stale_share": same.mean(),
"max_stale_run": int(runs.max()),
"max_stale_run_days": float(runs.max() * 15 / 60 / 24),
"n_resets_gt10": int((g["battery"].diff() > 10).sum()),
"max_drop": float(g["battery"].diff().min()),
"corr_density": g[["battery", "value"]].corr().iloc[0, 1],
}
)
battery = pd.DataFrame(battery_rows)
battery.to_csv(TAB / "battery_summary.csv", index=False)
# battery vs daily coverage/readings (core period)
daily_batt_cov = []
for sid, g in paired[paired.sensor_id.isin(core_ids)].groupby("sensor_id"):
g = g[(g.time >= PANEL_START) & (g.time < PANEL_END)].copy()
if g.empty:
continue
g["day"] = g["time"].dt.floor("1D")
daily = g.groupby("day").agg(
battery=("battery", "median"),
readings=("battery", "size"),
density=("value", "mean"),
)
daily_batt_cov.append(
{
"sensor_id": sid,
"days": len(daily),
"corr_battery_readings": daily[["battery", "readings"]].corr().iloc[0, 1],
"corr_battery_density": daily[["battery", "density"]].corr().iloc[0, 1],
"median_readings": daily["readings"].median(),
"median_battery": daily["battery"].median(),
}
)
battery_daily = pd.DataFrame(daily_batt_cov)
battery_daily.to_csv(TAB / "battery_vs_coverage.csv", index=False)
battery_panel = paired[
paired.sensor_id.isin(core_ids)
& (paired.time >= PANEL_START)
& (paired.time < PANEL_END)
]
panel_device_ids = [
s for s in core_ids
if len(dens[(dens.sensor_id == s) & (dens.time >= PANEL_START)]) > 0
]
battery_struct = {
"dens_rows": int(len(dens)),
"battery_rows": int(len(batt)),
"density_rows_with_battery_60s": int(paired.battery.notna().sum()),
"density_rows_without_battery_60s": int(paired.battery.isna().sum()),
"core_panel_density_rows": int(len(battery_panel)),
"core_panel_active_devices": int(len(panel_device_ids)),
"core_panel_expected_rows": int(len(panel_device_ids) * PANEL_SLOTS_TOTAL),
"core_panel_battery_coverage_pct": 100 * len(battery_panel) / (len(panel_device_ids) * PANEL_SLOTS_TOTAL),
}
# ------------------------------------------------------ location matrix
core = core.copy()
core["location"] = core["sensor_id"].map(LOC_MAP).fillna(core["name"])
loc_slot = (
core.groupby(["location", "slot"], as_index=False)
.agg(value=("value", "mean"), n=("value", "size"))
)
mat_all = loc_slot.pivot(index="slot", columns="location", values="value").sort_index()
locs = list(mat_all.columns)
panel_mat = mat_all[(mat_all.index >= PANEL_START) & (mat_all.index < PANEL_END)]
panel_slots_total = int((PANEL_END - PANEL_START) / pd.Timedelta(SLOT))
panel_mat = panel_mat.reindex(
pd.date_range(PANEL_START, PANEL_END, freq=SLOT, inclusive="left")
)
panel_mat.to_csv(TAB / "panel_15min_matrix.csv")
n_report = panel_mat.notna().sum(axis=1)
report_hist = n_report.value_counts().sort_index()
report_hist.to_csv(TAB / "locations_per_slot.csv", header=["slots"])
all8 = panel_mat.dropna()
all8_slots = len(all8)
any_slots = int((n_report > 0).sum())
loc_coverage = panel_mat.notna().mean()
all8_independence = float(loc_coverage.prod() * panel_slots_total)
monthly_coverage = (panel_mat.notna().resample("MS").mean() * 100).round(2)
monthly_coverage.to_csv(TAB / "monthly_coverage.csv")
# monthly diagnostics
month_rows = []
for per, g in panel_mat.groupby(panel_mat.index.to_period("M")):
c = g.notna().sum(axis=1)
cv_m = (g.std(axis=1) / g.mean(axis=1)).where((c >= 2) & (g.mean(axis=1) > 0))
r_m = g.corr().where(~np.eye(len(locs), dtype=bool)).stack().mean()
month_rows.append(
{
"month": str(per),
"slots": len(g),
"all8_slots": int(g.dropna().shape[0]),
"all8_pct": 100 * g.dropna().shape[0] / len(g),
"any_pct": 100 * (c > 0).mean(),
"mean_pairwise_r": r_m,
"median_cv": cv_m.median(),
}
)
monthly_diag = pd.DataFrame(month_rows)
monthly_diag.to_csv(TAB / "monthly_diagnostics.csv", index=False)
# aggregation sensitivity: how complete is the 8-location panel at coarser bins?
agg_rows = []
for freq, minutes in [("15min", 15), ("30min", 30), ("60min", 60), ("120min", 120)]:
tmp = core.copy()
tmp["bin"] = tmp["time"].dt.floor(freq)
b = tmp.pivot_table(index="bin", columns="location", values="value", aggfunc="mean")
b = b[(b.index >= PANEL_START) & (b.index < PANEL_END)]
expected = int((PANEL_END - PANEL_START) / pd.Timedelta(minutes=minutes))
cc = b.notna().sum(axis=1)
cm = (b.std(axis=1) / b.mean(axis=1)).where((cc >= 2) & (b.mean(axis=1) > 0))
rr = b.corr().where(~np.eye(len(locs), dtype=bool)).stack().mean() if b.notna().sum().min() > 2 else np.nan
agg_rows.append(
{
"bin_minutes": minutes,
"expected_bins": expected,
"bins_with_any": int((cc > 0).sum()),
"all8_bins": int(b.dropna().shape[0]),
"all8_pct": 100 * b.dropna().shape[0] / expected,
"mean_pairwise_r": rr,
"median_cv": cm.median(),
}
)
agg_sens = pd.DataFrame(agg_rows)
agg_sens.to_csv(TAB / "aggregation_sensitivity.csv", index=False)
# August outage vs battery: strong cross-device confound
aug_mask = (panel_mat.index.year == 2026) & (panel_mat.index.month == 8)
aug_cov = (panel_mat[aug_mask].notna().mean() * 100).rename("aug_coverage_pct")
aug_b = paired[
(paired.time >= pd.Timestamp("2026-08-01", tz="UTC"))
& (paired.time < pd.Timestamp("2026-09-01", tz="UTC"))
& (paired.sensor_id.isin(core_ids))
].copy()
aug_b["location"] = aug_b["sensor_id"].map(LOC_MAP).fillna(aug_b["name"])
aug_batt = aug_b.groupby("location")["battery"].agg(
aug_battery_median="median",
aug_battery_p10=lambda s: s.quantile(0.10),
aug_battery_p90=lambda s: s.quantile(0.90),
)
aug_tab = pd.concat([aug_cov, aug_batt], axis=1).dropna()
aug_tab.to_csv(TAB / "august_battery_coverage.csv")
aug_r = float(aug_tab["aug_coverage_pct"].corr(aug_tab["aug_battery_median"]))
# missingness is not MCAR: compare density on high- vs low-coverage days
panel_local = panel_mat.copy()
panel_local.index = panel_local.index.tz_convert(TZ)
dmean = panel_local.resample("1D").mean()
dcount = panel_local.resample("1D").count()
bias_rows = []
for c in locs:
hi = dmean.loc[dcount[c] >= 80, c].median()
lo = dmean.loc[dcount[c] < 50, c].median()
bias_rows.append(
{
"location": c,
"median_density_high_cov_days": hi,
"median_density_low_cov_days": lo,
"ratio_low_to_high": lo / hi if hi else np.nan,
"days_below_50pct": int((dcount[c] < 50).sum()),
}
)
coverage_bias = pd.DataFrame(bias_rows)
coverage_bias.to_csv(TAB / "coverage_bias.csv", index=False)
miss_hour = (panel_local.isna().groupby(panel_local.index.hour).mean() * 100).round(2)
miss_hour.to_csv(TAB / "missingness_by_hour.csv")
# pairwise correlation (pairwise complete observations)
def pairwise_corr(mat: pd.DataFrame, method: str = "pearson") -> tuple[pd.DataFrame, pd.DataFrame]:
cols = list(mat.columns)
r = pd.DataFrame(np.nan, index=cols, columns=cols, dtype=float)
nn = pd.DataFrame(0, index=cols, columns=cols, dtype=int)
for a in cols:
for b in cols:
sub = mat[[a, b]].dropna()
nn.loc[a, b] = len(sub)
if len(sub) >= 3:
x = sub.iloc[:, 0]
y = sub.iloc[:, 1]
if method == "spearman":
r.loc[a, b] = x.rank().corr(y.rank())
else:
r.loc[a, b] = x.corr(y)
return r, nn
pear, pear_n = pairwise_corr(panel_mat, "pearson")
spear, _ = pairwise_corr(panel_mat, "spearman")
logp, _ = pairwise_corr(np.log1p(panel_mat), "pearson")
pear.to_csv(TAB / "correlation_pearson.csv")
spear.to_csv(TAB / "correlation_spearman.csv")
logp.to_csv(TAB / "correlation_log1p.csv")
pear_n.to_csv(TAB / "correlation_pair_n.csv")
# detrended correlation: remove location-specific hour-of-week mean
det = panel_mat.copy()
how = pd.Series(det.index.tz_convert(TZ).hour * 7 + det.index.tz_convert(TZ).day_of_week, index=det.index)
for c in det.columns:
det[c] = det[c] - det[c].groupby(how).transform("mean")
det_corr, _ = pairwise_corr(det, "pearson")
det_corr.to_csv(TAB / "correlation_detrended.csv")
# daily-level robustness: collapses autocorrelation to ~143 observations
daily_for_corr = panel_mat.resample("1D").mean()
daily_cov_for_corr = panel_mat.resample("1D").count()
daily_for_corr = daily_for_corr.where(daily_cov_for_corr.min(axis=1) >= 0.8 * 96)
daily_corr, daily_n = pairwise_corr(daily_for_corr, "pearson")
daily_corr.to_csv(TAB / "correlation_daily.csv")
# cross-sensor dispersion at same slot
cnt = panel_mat.notna().sum(axis=1)
mean = panel_mat.mean(axis=1)
std = panel_mat.std(axis=1)
cv = (std / mean).where((cnt >= 2) & (mean > 0))
cv_by_hour = cv.groupby(cv.index.tz_convert(TZ).hour).median()
shares = panel_mat.div(panel_mat.sum(axis=1), axis=0)
share_cv = (shares.std(axis=1) / shares.mean(axis=1)).where(cnt >= 2)
cv_by_hour.to_csv(TAB / "cv_by_hour.csv", header=["median_cv"])
# daily ratio stability (daily mean per location, panel days)
daily_mean = panel_mat.resample("1D").mean()
daily_count = panel_mat.resample("1D").count()
good_days = daily_count.min(axis=1) >= 0.8 * 96
daily_good = daily_mean[good_days]
share = daily_good.div(daily_good.sum(axis=1), axis=0)
share_cv = share.std() / share.mean()
log_ratio = np.log(daily_good.replace(0, np.nan))
pair_sd = []
for i, a in enumerate(locs):
for b in locs[i + 1:]:
pair_sd.append((log_ratio[a] - log_ratio[b]).std())
ratio_summary = {
"good_days": int(good_days.sum()),
"panel_days": int(len(good_days)),
"median_daily_share_cv": float(share_cv.median()),
"max_daily_share_cv": float(share_cv.max()),
"median_pairwise_log_ratio_sd": float(np.nanmedian(pair_sd)),
"p90_pairwise_log_ratio_sd": float(np.nanpercentile(pair_sd, 90)),
"share_cv": share_cv.round(4).to_dict(),
}
# hourly / weekday profiles (local time)
pser = panel_mat.copy()
pser.index = pser.index.tz_convert(TZ)
hourly = pser.groupby(pser.index.hour).mean()
weekday = pser.groupby(pser.index.day_of_week).mean()
hourly.to_csv(TAB / "hourly_profile.csv")
weekday.to_csv(TAB / "weekday_profile.csv")
hourly_norm = hourly / hourly.mean()
weekend_ratio = {
c: float(
weekday.loc[[5, 6], c].mean() / weekday.loc[[0, 1, 2, 3, 4], c].mean()
)
for c in locs
}
peak_hour = hourly.idxmax().to_dict()
shape_corr, _ = pairwise_corr(hourly_norm, "pearson")
off = shape_corr.where(~np.eye(len(locs), dtype=bool)).stack()
hourly_shape_mean_r = float(off.mean())
# zero / low-count distribution
zero_rows = []
for sid, g in dens.groupby("sensor_id"):
zero_rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"n": len(g),
"zeros": int((g.value == 0).sum()),
"zero_pct": pct((g.value == 0).mean()),
"lt1_pct": pct((g.value < 1).mean()),
"lt5_pct": pct((g.value < 5).mean()),
"median_nonzero": g.loc[g.value > 0, "value"].median(),
"mean_nonzero": g.loc[g.value > 0, "value"].mean(),
"negative": int((g.value < 0).sum()),
"nan": int(g.value.isna().sum()),
}
)
zero = pd.DataFrame(zero_rows).sort_values("n", ascending=False)
zero.to_csv(TAB / "zero_lowcount.csv", index=False)
lowcount = {
"all_zero_pct": pct((dens.value == 0).mean(), 2),
"all_lt1_pct": pct((dens.value < 1).mean(), 2),
"all_lt5_pct": pct((dens.value < 5).mean(), 2),
"quantization_01": bool(np.allclose((dens.value * 10).round() / 10, dens.value)),
"share_multiple_0_7": float(np.mean(np.abs(dens.value / 0.7 - (dens.value / 0.7).round()) < 1e-6)),
}
# ------------------------------------------ value grid / quantization
quant_rows = []
for (sid, nm), g in dens.groupby(["sensor_id", "name"]):
v = g["value"]
mult07 = np.isclose(v / 0.7, np.round(v / 0.7), atol=1e-9)
integer = np.isclose(v, np.round(v), atol=1e-9)
mult05 = np.isclose(v / 0.5, np.round(v / 0.5), atol=1e-9)
mult01 = np.isclose(v / 0.1, np.round(v / 0.1), atol=1e-9)
if sid == "863f75b0":
period, quantum = "legacy", 1.0
elif sid in episodic_ids:
period, quantum = "episodic", 0.7
else:
period, quantum = "current", 0.7
quant_rows.append(
{
"sensor_id": sid,
"name": nm,
"period": period,
"n": len(g),
"first": g["time"].min().strftime("%Y-%m-%d"),
"last": g["time"].max().strftime("%Y-%m-%d"),
"min": v.min(),
"max": v.max(),
"n_unique": int(v.nunique()),
"min_positive": v[v > 0].min() if (v > 0).any() else np.nan,
"effective_quantum": quantum,
"pct_mult_0_7": 100 * mult07.mean(),
"pct_integer": 100 * integer.mean(),
"pct_mult_0_7_and_integer": 100 * (mult07 & integer).mean(),
"pct_mult_0_7_only": 100 * (mult07 & ~integer).mean(),
"pct_integer_only": 100 * (~mult07 & integer).mean(),
"pct_neither": 100 * (~mult07 & ~integer).mean(),
"pct_mult_0_5": 100 * mult05.mean(),
"pct_mult_0_1": 100 * mult01.mean(),
"mean": v.mean(),
"median": v.median(),
"p95": v.quantile(0.95),
"zero_pct": 100 * (v == 0).mean(),
}
)
quant = pd.DataFrame(quant_rows).sort_values(["period", "name", "first"])
quant.to_csv(TAB / "quantization_by_device.csv", index=False)
old = dens[dens.sensor_id == "863f75b0"]
new = dens[dens.sensor_id == "f8c19730"]
scale_rows = []
for label, g, quantum in [("legacy 863f75b0", old, 1.0), ("new f8c19730", new, 0.7)]:
scale_rows.append(
{
"series": label,
"n": len(g),
"first": g.time.min().isoformat(),
"last": g.time.max().isoformat(),
"min": g.value.min(),
"max": g.value.max(),
"mean": g.value.mean(),
"median": g.value.median(),
"p95": g.value.quantile(0.95),
"p99": g.value.quantile(0.99),
"zero_pct": 100 * (g.value == 0).mean(),
"n_unique": int(g.value.nunique()),
"effective_quantum": quantum,
}
)
scale = pd.DataFrame(scale_rows)
gap_days = (new.time.min() - old.time.max()).total_seconds() / 86400
scale["gap_after_days"] = [gap_days, np.nan]
scale.to_csv(TAB / "density01_scale_comparison.csv", index=False)
# ------------------------------------------------- exact cadence / jitter
cadence_jitter_rows = []
for sid, g in core.groupby("sensor_id"):
g = g.sort_values("time")
dt = g["time"].diff().dt.total_seconds().dropna()
dt = dt[dt > 0]
near = dt[(dt > 840) & (dt < 1020)]
cadence_jitter_rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"kind": "legacy" if sid == "863f75b0" else "current",
"n_intervals": len(dt),
"median_all_s": dt.median(),
"mean_all_s": dt.mean(),
"sd_all_s": dt.std(),
"median_near_s": near.median(),
"mean_near_s": near.mean(),
"sd_near_s": near.std(),
"iqr_low_s": near.quantile(0.25),
"iqr_high_s": near.quantile(0.75),
"p05_near_s": near.quantile(0.05),
"p95_near_s": near.quantile(0.95),
"share_900_5s_pct": 100 * ((near - 900).abs() <= 5).mean(),
"share_900_30s_pct": 100 * ((near - 900).abs() <= 30).mean(),
"share_900_60s_pct": 100 * ((near - 900).abs() <= 60).mean(),
"share_14_17min_pct": 100 * len(near) / len(dt),
}
)
cadence_jitter = pd.DataFrame(cadence_jitter_rows)
cadence_jitter.to_csv(TAB / "cadence_jitter.csv", index=False)
# ------------------------------------------------------ integrity checks
integrity = pd.DataFrame(
[
("exact_duplicate_rows", int(raw.duplicated().sum())),
("duplicate_time_sensor_type", int(raw.duplicated(["time", "sensor_id", "type"]).sum())),
("duplicate_time_sensor", int(raw.duplicated(["time", "sensor_id"]).sum())),
("density_negative", int((dens.value < 0).sum())),
("density_nan", int(dens.value.isna().sum())),
("battery_negative", int((batt.value < 0).sum())),
("battery_over_100", int((batt.value > 100).sum())),
("battery_nan", int(batt.value.isna().sum())),
("core_intervals_lt_60s", int((core.groupby("sensor_id")["time"].diff().dt.total_seconds() < 60).sum())),
],
columns=["check", "count"],
)
integrity.to_csv(TAB / "integrity_checks.csv", index=False)
# -------------------------------------------------------------- anomalies
gap_rows = []
for sid, g in dens.groupby("sensor_id"):
g = g.sort_values("time")
dt = g["time"].diff().dt.total_seconds() / 3600
for i in np.where(dt > 24)[0][:10]:
gap_rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"gap_start": g["time"].iloc[i - 1].strftime("%Y-%m-%d %H:%M"),
"gap_end": g["time"].iloc[i].strftime("%Y-%m-%d %H:%M"),
"gap_hours": dt.iloc[i],
}
)
gaps = pd.DataFrame(gap_rows).sort_values("gap_hours", ascending=False)
gaps.to_csv(TAB / "large_gaps.csv", index=False)
spike_rows = []
spike_detail = []
flat_rows = []
for sid, g in core.groupby("sensor_id"):
g = g.sort_values("time").reset_index(drop=True)
med = g["value"].rolling(9, center=True, min_periods=5).median()
ratio = g["value"] / (med + 0.7)
mask = (ratio >= 5) & (g["value"] >= 10)
n_spike = int(mask.sum())
spike_rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"spikes_ratio5": n_spike,
"max_ratio": float(ratio.max()),
"spike_times": ", ".join(g.loc[mask, "time"].dt.strftime("%Y-%m-%d %H:%M").head(3)),
}
)
for pos in np.where(mask)[0]:
spike_detail.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"time": g["time"].iloc[pos].strftime("%Y-%m-%d %H:%M"),
"value": g["value"].iloc[pos],
"prev": g["value"].iloc[pos - 1] if pos > 0 else np.nan,
"next": g["value"].iloc[pos + 1] if pos + 1 < len(g) else np.nan,
"ratio_to_local_median": ratio.iloc[pos],
}
)
run_id = g["value"].ne(g["value"].shift()).cumsum()
runs = g.groupby(run_id).agg(
n=("value", "size"), value=("value", "first"), start=("time", "first")
)
longest = runs.loc[runs["n"].idxmax()]
flat_rows.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"adjacent_equal_share": (g["value"].diff() == 0).mean(),
"runs_ge3": int((runs["n"] >= 3).sum()),
"max_run": int(longest["n"]),
"max_run_hours": float(longest["n"] * 15 / 60),
"max_run_value": float(longest["value"]),
"max_run_start": longest["start"].strftime("%Y-%m-%d %H:%M"),
"share_in_runs_ge3": float(runs.loc[runs["n"] >= 3, "n"].sum() / len(g)),
}
)
spikes = pd.DataFrame(spike_rows)
spike_detail = pd.DataFrame(spike_detail).sort_values("ratio_to_local_median", ascending=False)
flat = pd.DataFrame(flat_rows)
spikes.to_csv(TAB / "spikes.csv", index=False)
spike_detail.head(15).to_csv(TAB / "top_spikes.csv", index=False)
flat.to_csv(TAB / "flatlines.csv", index=False)
near_dup = []
for sid, g in core.groupby("sensor_id"):
g = g.sort_values("time").reset_index(drop=True)
dt = g["time"].diff().dt.total_seconds()
for pos in np.where(dt < 60)[0]:
near_dup.append(
{
"sensor_id": sid,
"name": g["name"].iloc[0],
"prev_time": g["time"].iloc[pos - 1].strftime("%Y-%m-%d %H:%M:%S"),
"time": g["time"].iloc[pos].strftime("%Y-%m-%d %H:%M:%S"),
"delta_s": dt.iloc[pos],
"prev_value": g["value"].iloc[pos - 1],
"value": g["value"].iloc[pos],
}
)
near_dup = pd.DataFrame(near_dup)
near_dup.to_csv(TAB / "near_duplicate_intervals.csv", index=False)
# --------------------------------------------------------------- summary
summary = {
"rows": int(len(raw)),
"density_rows": int(len(dens)),
"battery_rows": int(len(batt)),
"sensors_total": int(raw.sensor_id.nunique()),
"sensors_core": int(len(core_ids)),
"locations_core": int(len(locs)),
"sensors_episodic": int(len(episodic_ids)),
"time_min": raw.time.min().isoformat(),
"time_max": raw.time.max().isoformat(),
"panel_start": PANEL_START.isoformat(),
"panel_end": PANEL_END.isoformat(),
"panel_days": int((PANEL_END - PANEL_START).days),
"panel_slots_total": panel_slots_total,
"all8_slots": all8_slots,
"all8_pct_panel": pct(all8_slots / panel_slots_total, 2),
"all8_pct_any": pct(all8_slots / any_slots, 2),
"all8_expected_independence": round(all8_independence, 1),
"all8_pct_30min": float(agg_sens.loc[agg_sens.bin_minutes == 30, "all8_pct"].iloc[0]),
"all8_pct_60min": float(agg_sens.loc[agg_sens.bin_minutes == 60, "all8_pct"].iloc[0]),
"all8_pct_120min": float(agg_sens.loc[agg_sens.bin_minutes == 120, "all8_pct"].iloc[0]),
"any_slots": any_slots,
"locations_per_slot_hist": {int(k): int(v) for k, v in report_hist.items()},
"monthly_coverage_min_pct": float(monthly_coverage.min().min()),
"monthly_coverage_median_pct": float(monthly_coverage.stack().median()),
"all_zero_pct": lowcount["all_zero_pct"],
"all_lt1_pct": lowcount["all_lt1_pct"],
"all_lt5_pct": lowcount["all_lt5_pct"],
"share_multiple_0_7": round(lowcount["share_multiple_0_7"], 4),
"quantization_summary": {
"devices_on_0_7_grid": int((np.isclose(quant["pct_mult_0_7"], 100)).sum()),
"devices_integer_only": int(((np.isclose(quant["pct_integer"], 100)) & (quant["pct_mult_0_7"] < 100)).sum()),
"devices_with_neither": int((quant["pct_neither"] > 0).sum()),
"pct_neither_overall": float((quant["pct_neither"] * quant["n"]).sum() / quant["n"].sum()),
"pct_integer_overall": float((quant["pct_integer"] * quant["n"]).sum() / quant["n"].sum()),
},
"density01_old": {
k: float(v) if isinstance(v, (int, float, np.integer, np.floating)) else str(v)
for k, v in scale.iloc[0].drop("gap_after_days").items()
},
"density01_new": {
k: float(v) if isinstance(v, (int, float, np.integer, np.floating)) else str(v)
for k, v in scale.iloc[1].drop("gap_after_days").items()
},
"density01_gap_days": float(gap_days),
"cadence_legacy_median_s": float(
cadence_jitter.loc[cadence_jitter.sensor_id == "863f75b0", "median_near_s"].iloc[0]
),
"cadence_new_density_median_s": float(
cadence_jitter.loc[cadence_jitter.sensor_id == "f8c19730", "median_near_s"].iloc[0]
),
"corr_pearson_mean_offdiag": float(
pear.where(~np.eye(len(locs), dtype=bool)).stack().mean()
),
"corr_pearson_min_offdiag": float(
pear.where(~np.eye(len(locs), dtype=bool)).stack().min()
),
"corr_pearson_max_offdiag": float(
pear.where(~np.eye(len(locs), dtype=bool)).stack().max()
),
"corr_spearman_mean_offdiag": float(
spear.where(~np.eye(len(locs), dtype=bool)).stack().mean()
),
"corr_log1p_mean_offdiag": float(
logp.where(~np.eye(len(locs), dtype=bool)).stack().mean()
),
"corr_detrended_mean_offdiag": float(
det_corr.where(~np.eye(len(locs), dtype=bool)).stack().mean()
),
"corr_daily_mean_offdiag": float(
daily_corr.where(~np.eye(len(locs), dtype=bool)).stack().mean()
),
"daily_corr_days": int(daily_for_corr.dropna().shape[0]),
"cv_median": float(cv.median()),
"cv_p90": float(cv.quantile(0.90)),
"cv_p95": float(cv.quantile(0.95)),
"share_cv_median": float(share_cv.median()),
"share_cv_p95": float(share_cv.quantile(0.95)),
"hourly_shape_mean_r": hourly_shape_mean_r,
"peak_hour_local": {k: int(v) for k, v in peak_hour.items()},
"weekend_weekday_ratio": {k: round(v, 3) for k, v in weekend_ratio.items()},
"ratio_stability": ratio_summary,
"coverage_bias_ratio_median": float(coverage_bias["ratio_low_to_high"].median()),
"coverage_bias_ratio_min": float(coverage_bias["ratio_low_to_high"].min()),
"coverage_bias_ratio_max": float(coverage_bias["ratio_low_to_high"].max()),
"august_coverage_battery_r": aug_r,
"duplicates": dup,
"battery_structural": battery_struct,
"battery_missing_pct": pct(paired.battery.isna().mean(), 2),
"battery_median_all": float(paired.battery.median()),
"battery_share_lt20": pct((paired.battery < 20).mean(), 1),
"battery_share_zero": pct((paired.battery == 0).mean(), 1),
"battery_offset_median_s": float(paired.offset_s.median()),
"battery_offset_p99_s": float(paired.offset_s.quantile(0.99)),
"near_duplicate_intervals": int(len(near_dup)),
"max_flatline_run": int(flat["max_run"].max()),
"max_flatline_run_hours": float(flat["max_run_hours"].max()),
"max_value": float(dens.value.max()),
"max_value_sensor": dens.loc[dens.value.idxmax(), "name"],
"max_value_time": dens.loc[dens.value.idxmax(), "time"].isoformat(),
}
with open(TAB / "summary.json", "w") as fh:
json.dump(summary, fh, indent=2)
# ------------------------------------------------------------- figures
# 1. per-location time series
daily = panel_mat.resample("1D").mean()
dcount = panel_mat.resample("1D").count()
daily = daily.where(dcount >= 80) # >=83% of 96 slots; excludes partial days
fig, axes = plt.subplots(4, 2, figsize=(11, 8), sharex=True)
for ax, c in zip(axes.ravel(), locs):
ax.plot(daily.index, daily[c], color="#4c72b0", lw=0.8, alpha=0.55, label="daily mean")
roll = daily[c].rolling(7, min_periods=5).mean()
roll = roll.where(daily[c].notna().rolling(3, min_periods=1).sum() >= 2)
ax.plot(daily.index, roll, color="#c44e52", lw=1.4, label="7-day mean")
ax.set_title(c)
ax.set_ylabel("mean density / 15 min")
ax.xaxis.set_major_formatter(mdates.DateFormatter("%b"))
axes[0, 0].legend(fontsize=6, loc="upper left")
fig.suptitle("Per-location density, 15-min readings aggregated to daily means", y=1.01)
fig.tight_layout()
fig.savefig(FIG / "fig_time_series.png", bbox_inches="tight")
plt.close(fig)
# 2. correlation heatmaps
fig, axes = plt.subplots(1, 3, figsize=(15, 4.6))
for ax, (m, title) in zip(
axes,
[(pear, "Pearson (raw)"), (spear, "Spearman (rank)"), (det_corr, "Pearson (hour-of-week removed)")],
):
im = ax.imshow(m.values, vmin=-0.2, vmax=1, cmap="RdYlBu_r")
ax.set_xticks(range(len(locs)))
ax.set_yticks(range(len(locs)))
ax.set_xticklabels(locs, rotation=60, ha="right", fontsize=6)
if ax is axes[0]:
ax.set_yticklabels(locs, fontsize=6)
else:
ax.set_yticklabels([])
ax.set_title(title)
for i in range(len(locs)):
for j in range(len(locs)):
v = m.values[i, j]
ax.text(j, i, f"{v:.2f}", ha="center", va="center", fontsize=5.5,
color="white" if abs(v) > 0.65 else "black")
ax.grid(False)
cax = fig.add_axes([0.915, 0.22, 0.012, 0.56])
fig.colorbar(im, cax=cax, label="correlation")
fig.suptitle("Cross-location correlation at matching 15-min slots (pairwise complete)", y=1.02)
fig.subplots_adjust(wspace=0.10, right=0.90)
fig.savefig(FIG / "fig_correlation_heatmap.png", bbox_inches="tight")
plt.close(fig)
# 3. missingness heatmap (location x day, % of 96 slots present)
cov_daily = panel_mat.notna().resample("1D").sum() / 96 * 100
fig, ax = plt.subplots(figsize=(12, 3.6))
im = ax.imshow(cov_daily.T.values, aspect="auto", cmap="viridis", vmin=0, vmax=100)
ax.set_yticks(range(len(locs)))
ax.set_yticklabels(locs, fontsize=7)
tick = np.arange(0, len(cov_daily), 14)
ax.set_xticks(tick)
ax.set_xticklabels([d.strftime("%d %b") for d in cov_daily.index[tick]], rotation=45, ha="right", fontsize=7)
ax.set_title("Daily slot coverage: % of 96 possible 15-min slots with a reading")
ax.grid(False)
fig.colorbar(im, ax=ax, label="% of 96 slots", pad=0.01)
fig.tight_layout()
fig.savefig(FIG / "fig_missingness_heatmap.png", bbox_inches="tight")
plt.close(fig)
# 4. distribution
fig, axes = plt.subplots(1, 3, figsize=(14, 3.9))
v = dens["value"]
axes[0].hist(v, bins=np.arange(0, 101, 0.7), color="#4c72b0")
axes[0].set_yscale("log")
axes[0].set_xlabel("density value (15-min count)")
axes[0].set_ylabel("readings (log scale)")
axes[0].set_title("Distribution, 0-100")
axes[0].axvline(1, color="red", ls="--", lw=0.8)
axes[0].text(1.5, axes[0].get_ylim()[1] * 0.5, "<1", color="red", fontsize=7)
nz = np.sort(v[v > 0].values)
ccdf = 1 - np.arange(len(nz)) / len(nz)
axes[1].loglog(nz, ccdf, color="#c44e52", lw=1)
axes[1].set_xlabel("density value > 0")
axes[1].set_ylabel("P(X >= x)")
axes[1].set_title("CCDF (log-log)")
core_dev = coverage[coverage.role == "core"].sort_values("mean")
data = [core.loc[core.sensor_id == sid, "value"].values for sid in core_dev.sensor_id]
short = {
"863f75b0": "density-01 old",
"f8c19730": "density-01 new",
"ba71377f": "soleo",
"00091a5f": "goetzentuem",
"8d7eef22": "marahaus",
"d4cce74b": "neckarufer",
"0f8519cf": "hagenbuchersee-02",
"e52c55f3": "hagenbuchersee-01",
"f4928534": "stadtgalerie",
}
labels = [short.get(sid, nm.replace("hn-", "")) for sid, nm in zip(core_dev.sensor_id, core_dev.name)]
axes[2].boxplot(data, tick_labels=labels, showfliers=False)
axes[2].set_yscale("symlog", linthresh=1)
axes[2].set_ylabel("density value (symlog)")
axes[2].tick_params(axis="x", rotation=55, labelsize=6)
axes[2].set_title("Per-device distribution")
fig.tight_layout()
fig.savefig(FIG / "fig_distribution.png", bbox_inches="tight")
plt.close(fig)
# 4b. quantization grid and density-01 scale discontinuity
fig, axes = plt.subplots(1, 2, figsize=(13, 4.2), gridspec_kw={"width_ratios": [1.6, 1]})
q = quant.copy()
shortq = {
"863f75b0": "density-01 legacy",
"f8c19730": "density-01 new",
"ba71377f": "soleo",
"00091a5f": "goetzentuem",
"8d7eef22": "marahaus",
"d4cce74b": "neckarufer",
"0f8519cf": "hagenbuchersee-02",
"e52c55f3": "hagenbuchersee-01",
"f4928534": "stadtgalerie",
"d3304b39": "bollwerksturm",
"2aadbeed": "maker-fair",
"062fe216": "niklas-01",
}
q["label"] = q["sensor_id"].map(shortq).fillna(q["name"])
q = q.sort_values(["period", "name"]).reset_index(drop=True)
y = np.arange(len(q))
left = np.zeros(len(q))
for col, color, lab in [
("pct_mult_0_7_only", "#4c72b0", "0.7 grid, non-integer"),
("pct_mult_0_7_and_integer", "#a6bddb", "0.7 grid and integer"),
("pct_integer_only", "#dd8452", "integer grid only"),
("pct_neither", "#c44e52", "neither"),
]:
vals = q[col].values
axes[0].barh(y, vals, left=left, color=color, label=lab, height=0.72)
left = left + vals
axes[0].set_yticks(y)
axes[0].set_yticklabels(q["label"], fontsize=6)
axes[0].set_xlim(0, 100)
axes[0].set_xlabel("share of readings (%)")
axes[0].set_title("Value-grid membership by device")
axes[0].legend(fontsize=6, loc="lower right", framealpha=0.9)
axes[0].grid(axis="x", alpha=0.25)
old_vals = np.sort(old["value"].unique())
new_vals = np.sort(new["value"].unique())
axes[1].plot(old_vals[old_vals <= 7], np.zeros((old_vals <= 7).sum()), "|", ms=18, color="#dd8452", label="legacy: 1.0 grid")
axes[1].plot(new_vals[new_vals <= 7], np.ones((new_vals <= 7).sum()), "|", ms=18, color="#4c72b0", label="new: 0.7 grid")
axes[1].set_yticks([0, 1])
axes[1].set_yticklabels(["863f75b0\n(legacy)", "f8c19730\n(new)"], fontsize=7)
axes[1].set_xlabel("reported value, 0-7 (zoomed)")
axes[1].set_xlim(-0.2, 7.2)
axes[1].set_ylim(-0.5, 1.5)
axes[1].set_title("Reported value ladders, density-01")
axes[1].legend(fontsize=6, loc="center right")
axes[1].grid(axis="x", alpha=0.25)
fig.tight_layout()
fig.savefig(FIG / "fig_quantization.png", bbox_inches="tight")
plt.close(fig)
fig, ax = plt.subplots(figsize=(9, 3.4))
for sid, color, lab in [("863f75b0", "#dd8452", "863f75b0 (legacy, 1.0 grid)"),
("f8c19730", "#4c72b0", "f8c19730 (new, 0.7 grid)")]:
g = dens[dens.sensor_id == sid].set_index("time")["value"].resample("1D").agg(["mean", "count"])
g = g[g["count"] >= 48]
ax.plot(g.index, g["mean"], color=color, lw=1.0, label=lab)
gap_start = old["time"].max()
gap_end = new["time"].min()
ax.axvspan(gap_start, gap_end, color="grey", alpha=0.18)
ax.text(gap_start + (gap_end - gap_start) / 2, ax.get_ylim()[1] * 0.55,
"35.2-day gap", ha="center", fontsize=7, color="dimgrey")
ax.set_ylabel("daily mean density")
ax.set_title("density-01 before and after the device replacement")
ax.legend(fontsize=7)
fig.tight_layout()
fig.savefig(FIG / "fig_density01_swap.png", bbox_inches="tight")
plt.close(fig)
# 5. cadence
fig, axes = plt.subplots(1, 2, figsize=(11, 3.8))
axes[0].hist(intervals["minutes"], bins=np.arange(0, 61, 1), color="#55a868")
axes[0].set_yscale("log")
axes[0].set_xlabel("inter-reading interval (minutes)")
axes[0].set_ylabel("count (log scale)")
axes[0].set_title("Cadence, all core devices, intervals <=60 min")
cad = cadence.set_index("name")
y = np.arange(len(cad))
axes[1].errorbar(
cad["median_min"], y,
xerr=[cad["median_min"] - cad["p25_min"], cad["p75_min"] - cad["median_min"]],
fmt="o", color="#4c72b0", ecolor="#999999", capsize=2,
)
for yy, (_, r) in zip(y, cad.iterrows()):
axes[1].text(16.6, yy, f"{100*r['share_28_34min']:.0f}% at ~30 min", va="center", fontsize=5.5)
axes[1].set_yticks(y)
axes[1].set_yticklabels(cad.index, fontsize=6)
axes[1].set_xlim(14.8, 21.5)
axes[1].set_xlabel("median interval, min (bar = IQR)")
axes[1].set_title("Nominal cadence by device")
fig.tight_layout()
fig.savefig(FIG / "fig_cadence.png", bbox_inches="tight")
plt.close(fig)
# 6. hourly / weekday profile
weekday_norm = weekday / weekday.mean()
fig, axes = plt.subplots(1, 2, figsize=(12, 3.8))
for c in locs:
axes[0].plot(hourly_norm.index, hourly_norm[c], marker="o", ms=2, lw=1, label=c)
axes[1].plot(weekday_norm.index, weekday_norm[c], marker="o", ms=2, lw=1, label=c)
axes[0].set_xlabel("hour of day (Europe/Berlin)")
axes[0].set_ylabel("mean / location mean")
axes[0].set_title("Normalized hourly profile")
axes[0].set_xticks(range(0, 24, 2))
axes[1].set_xlabel("day of week")
axes[1].set_ylabel("mean / location mean")
axes[1].set_title("Normalized weekday profile")
axes[1].set_xticks(range(7))
axes[1].set_xticklabels(["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"])
axes[1].legend(fontsize=5, ncol=2)
fig.tight_layout()
fig.savefig(FIG / "fig_hourly_profile.png", bbox_inches="tight")
plt.close(fig)
# 7. battery
fig, axes = plt.subplots(1, 3, figsize=(15, 3.8))
for sid, g in paired[paired.sensor_id.isin(core_ids)].groupby("sensor_id"):
g = g.set_index("time").sort_index()
daily_b = g["battery"].resample("1D").median()
axes[0].plot(daily_b.index, daily_b.values, lw=0.8, label=DEVICE_LABEL.get(sid, g["name"].iloc[0]))
axes[0].set_ylabel("battery (%)")
axes[0].set_title("Daily median battery level")
axes[0].legend(fontsize=5, ncol=2)
axes[0].xaxis.set_major_formatter(mdates.DateFormatter("%b"))
axes[1].hist(paired["battery"].dropna(), bins=np.arange(0, 102, 2), color="#8172b3")
axes[1].set_xlabel("battery (%)")
axes[1].set_ylabel("readings")
axes[1].set_title("Battery distribution (all paired readings)")
short_aug = {
"density-01": "density-01", "hn-goetzentuem-01": "goetz",
"hn-hagenbuchersee-01": "hagen-01", "hn-hagenbuchersee-02": "hagen-02",
"hn-marahaus-01": "marahaus", "hn-neckarufer-01": "neckar",
"hn-soleo-01": "soleo", "hn-stadtgalerie-01": "stadt",
}
axes[2].scatter(aug_tab["aug_battery_median"], aug_tab["aug_coverage_pct"], color="#4c72b0")
for lab, row in aug_tab.iterrows():
axes[2].annotate(short_aug.get(lab, lab), (row["aug_battery_median"], row["aug_coverage_pct"]),
fontsize=5.5, xytext=(2, 2), textcoords="offset points")
axes[2].set_xlabel("August median battery (%)")
axes[2].set_ylabel("August slot coverage (%)")
axes[2].set_title(f"August coverage vs battery (r = {aug_r:.2f})")
fig.tight_layout()
fig.savefig(FIG / "fig_battery.png", bbox_inches="tight")
plt.close(fig)
print(json.dumps(summary, indent=2))
print("\nWrote tables to", TAB)
print("Wrote figures to", FIG)
if __name__ == "__main__":
main()