Score a forecast only when the relay matched its plan in time, not just in share

Reproduced: a forecast planning 15 minutes of heat was scored although the
relay was on for 12 (the 0.4.1 rule allowed 20 % per 15-minute block), and
the same on-time placed elsewhere in a block passed as well.

- The relay's on-time per poll interval, integrated at its own switch
  times, is compared with the plan block by block up to each horizon: the
  deviation is the time the relay spent in another state than planned.
  Unobserved relay time counts as possible deviation.
- Matched (at most 2 minutes, worst case): Forecast Error figures.
  Approximate (at most 5 minutes, at most 2 of them unobserved): separate
  approximate_stats. Counterfactual (more than 5 minutes seen): counted, not
  scored. Incomplete otherwise, by reason: relay history, assumed inputs at
  forecast time (an unknown window or electrical heat), no poll near the
  due time, or a missing reading.
- 2 minutes of unplanned boiler heat move a room by roughly half a 0.1 °C
  sensor step for the learned heat coefficients (0.05-0.12 °C per 5
  minutes); 5 minutes by about one step.
- Stored ledger statistics are versioned (2); 0.4.1 statistics used the
  looser rule and are dropped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

custom_components/heating_mpc/coordinator.py

@@ -895,9 +895,16 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             _LOGGER.warning("MPC prediction failed: %s", err)
             status["status"] = "prediction_error"
             return status
+        # Inputs the forecast had to assume: such forecasts are not compared with reality.
+        uncertain = [f"window: {name}" for room in vals
+                     for name in (vent_inputs.get(room) or {}).get("unknown_windows", [])]
+        uncertain += [f"electrical heat: {self.zones[room].name}" for room in vals
+                      if internal[room] is None and room in self.extended
+                      and ZoneModel(self.learners[room], self.extended[room]).uses_extended]
         self._forecasts.record(now.timestamp(), vals, output["trajectory_15min"], output["plan_15min"],
                                self._revision(), {
             "strategy": "mpc_plan", "outside": outside, "wind": weather.get("wind"),
+            "uncertain_inputs": uncertain,
             "internal_half_life_minutes": float(self.config.get(CONF_INTERNAL_HALF_LIFE, 30)),
             "window_open_minutes": float(self.config[CONF_WINDOW_OPEN_MINUTES]),
             "internal_kw": {room: internal[room] for room in vals},

custom_components/heating_mpc/evaluation.py

@@ -371,11 +371,24 @@ def _stats(values: list[float], horizon: int, zones: int = 1) -> dict[str, Any]:
     }
 
 
-# A forecast's plan counts as executed when, in every 15-minute block, the share
-# of time the relay was on is within this of what the plan assumed (0 or 1).
-PLAN_TOLERANCE = 0.2
+# How far the relay may depart from a forecast's plan (seconds in another state than
+# planned, up to the horizon) for the forecast to be compared with what happened.
+# d seconds of boiler heat that the plan did not assume (or the reverse) move the
+# room by roughly heat x d / 300, where heat is the learned boiler coefficient
+# (typically 0.05-0.12 °C per 5 minutes with warm radiators). Up to 2 minutes keeps
+# that below half a typical 0.1 °C sensor step and absorbs timestamp jitter;
+# matched forecasts make the Forecast Error figures. Up to 5 minutes (one poll) is
+# about one sensor step: kept apart as approximate. Beyond, the relay did
+# something else: the forecast is counterfactual and is not scored. Relay time
+# that was not observed counts as possible deviation, and more than 2 minutes of it
+# leaves the forecast incomplete rather than approximate.
+MATCH_SECONDS = 120
+APPROXIMATE_SECONDS = 300
 DUE_TOLERANCE_SECONDS = 150  # half a poll: the poll closest to the forecast's due time
-VERDICTS = ("counterfactual", "unverifiable", "missing")
+VERDICTS = ("matched", "approximate", "counterfactual", "incomplete")
+# Why a forecast (or, for a missing reading, a zone's forecast) could not be compared.
+INCOMPLETE_REASONS = ("relay_history", "uncertain_inputs", "no_poll", "reading")
+LEDGER_VERSION = 2  # 1 (0.4.1) accepted plans within 20 % per 15-minute block
 
 
 class OperationalForecastLedger:
@@ -383,10 +396,10 @@ class OperationalForecastLedger:
 
     Nothing is replaced with hindsight: the start temperatures, trajectory, boiler
     plan and assumptions are those of the moment the forecast was made. Because the
-    plan is the MPC's proposal while Generic Thermostat controls the relay, a forecast
-    is only scored when the relay actually followed the plan up to that horizon;
-    otherwise it is counted as counterfactual. Pending forecasts live in memory;
-    the statistics are saved.
+    plan is the MPC's proposal while Generic Thermostat controls the relay, a
+    forecast is compared only when the relay actually followed that plan (see
+    MATCH_SECONDS), and not at all when it was made with an input it had to
+    assume. Pending forecasts live in memory; the statistics are saved.
     """
 
     def __init__(self, saved: Any = None) -> None:
@@ -394,8 +407,9 @@ class OperationalForecastLedger:
         # (interval end, seconds the relay was on or None if unknown, interval seconds)
         self._relay: deque[tuple[float, float | None, float]] = deque(maxlen=48)
         # UTC day -> horizon -> zone -> [count, sum error, sum |error|, sum error^2, sum |persistence error|]
-        self._days: dict[str, dict[str, dict[str, list[float]]]] = {}
-        # UTC day -> horizon -> verdict -> count of forecasts not scored, and why
+        self._days: dict[str, dict[str, dict[str, list[float]]]] = {}       # matched forecasts
+        self._approximate: dict[str, dict[str, dict[str, list[float]]]] = {}
+        # UTC day -> horizon -> verdict (incomplete ones as "incomplete:<reason>") -> count
         self._verdicts: dict[str, dict[str, dict[str, int]]] = {}
         self.load(saved)
 
@@ -420,20 +434,18 @@ class OperationalForecastLedger:
                 forecast["done"].add(horizon)
                 if index >= len(forecast["plan"]):
                     continue  # beyond this forecast's horizon
-                if now > due + DUE_TOLERANCE_SECONDS:
-                    self._count(day, horizon, "missing")  # no poll near the due time
-                    continue
-                verdict = self._verify(forecast, horizon)
-                if verdict:
-                    self._count(day, horizon, verdict)
+                verdict = self._classify(forecast, horizon, now)
+                self._count(day, horizon, verdict)
+                if verdict not in ("matched", "approximate"):
                     continue
+                store = self._days if verdict == "matched" else self._approximate
                 for zone, path in forecast["trajectory"].items():
                     measured, start = actual.get(zone), forecast["start"].get(zone)
                     if measured is None or start is None or len(path) <= index:
-                        self._count(day, horizon, "missing")
+                        self._count(day, horizon, "incomplete:reading")
                         continue
                     error = path[index] - measured
-                    bucket = self._days.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
+                    bucket = store.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
                         zone, [0, 0.0, 0.0, 0.0, 0.0])
                     bucket[0] += 1
                     bucket[1] += error
@@ -443,38 +455,62 @@ class OperationalForecastLedger:
             if len(forecast["done"]) == len(HORIZONS) or now > forecast["time"] + (HORIZONS[-1] + 5) * 60:
                 self._pending.remove(forecast)
         oldest = (datetime.fromtimestamp(now, timezone.utc).date() - timedelta(days=WINDOW_DAYS - 1)).isoformat()
-        for store in (self._days, self._verdicts):
+        for store in (self._days, self._approximate, self._verdicts):
             for old in [d for d in store if d < oldest]:
                 del store[old]
 
-    def _verify(self, forecast: dict[str, Any], horizon: int) -> str | None:
-        """None when the relay followed the plan up to `horizon`, else the reason."""
-        for block in range(horizon // 15):
-            start = forecast["time"] + block * 900
-            end = start + 900
-            on = covered = 0.0
-            for interval_end, on_seconds, seconds in self._relay:
-                overlap = min(end, interval_end) - max(start, interval_end - seconds)
-                if overlap <= 0:
-                    continue
-                if on_seconds is None:
-                    return "unverifiable"
-                on += on_seconds * overlap / seconds
-                covered += overlap
-            if covered < 0.8 * 900:
-                return "unverifiable"
-            if abs(on / covered - forecast["plan"][block]) > PLAN_TOLERANCE:
-                return "counterfactual"
-        return None
+    def _classify(self, forecast: dict[str, Any], horizon: int, now: float) -> str:
+        if now > forecast["time"] + horizon * 60 + DUE_TOLERANCE_SECONDS:
+            return "incomplete:no_poll"
+        if forecast["assumptions"].get("uncertain_inputs"):
+            return "incomplete:uncertain_inputs"
+        deviation, unobserved = self.deviation(forecast, horizon)
+        if deviation > APPROXIMATE_SECONDS:
+            return "counterfactual"  # whatever happened in the unobserved part
+        # Otherwise judged by the worst case: unobserved time may all have deviated.
+        if deviation + unobserved <= MATCH_SECONDS:
+            return "matched"
+        if deviation + unobserved <= APPROXIMATE_SECONDS and unobserved <= MATCH_SECONDS:
+            return "approximate"
+        return "incomplete:relay_history"
+
+    def deviation(self, forecast: dict[str, Any], horizon: int) -> tuple[float, float]:
+        """Seconds up to `horizon` the relay was seen in another state than planned,
+        and seconds its state is not known.
+
+        Compared per 15-minute plan block, from the relay's on-time per poll interval,
+        itself integrated at the relay's own switch times. So timing counts: the same
+        total on-time at other times than planned is a deviation, while a switch a few
+        seconds off a block boundary is not.
+        """
+        start, end = forecast["time"], forecast["time"] + horizon * 60
+        deviation = covered = unobserved = 0.0
+        for interval_end, on_seconds, seconds in self._relay:
+            lo, hi = max(start, interval_end - seconds), min(end, interval_end)
+            if hi <= lo:
+                continue
+            covered += hi - lo
+            if on_seconds is None:
+                unobserved += hi - lo
+                continue
+            share = on_seconds / seconds
+            t = lo
+            while t < hi:  # split at plan block boundaries
+                block = min(int((t - start) // 900), len(forecast["plan"]) - 1)
+                part_end = min(hi, start + (block + 1) * 900)
+                deviation += (part_end - t) * (share if not forecast["plan"][block] else 1.0 - share)
+                t = part_end
+        return deviation, unobserved + max(0.0, (end - start) - covered)
 
     def _count(self, day: str, horizon: int, verdict: str) -> None:
         counts = self._verdicts.setdefault(day, {}).setdefault(str(horizon), {})
         counts[verdict] = counts.get(verdict, 0) + 1
 
     def forget(self, zone: str) -> None:
-        for by_horizon in self._days.values():
-            for by_zone in by_horizon.values():
-                by_zone.pop(zone, None)
+        for store in (self._days, self._approximate):
+            for by_horizon in store.values():
+                for by_zone in by_horizon.values():
+                    by_zone.pop(zone, None)
         for forecast in self._pending:
             forecast["trajectory"].pop(zone, None)
 
@@ -485,58 +521,70 @@ class OperationalForecastLedger:
     def summary(self, zones: list[str]) -> dict[int, dict[str, Any]]:
         result = {}
         for horizon in HORIZONS:
-            per_zone: dict[str, list[float]] = {}
-            verdicts = {verdict: 0 for verdict in VERDICTS}
-            for by_horizon in self._days.values():
-                for zone, values in by_horizon.get(str(horizon), {}).items():
-                    if zone in zones:
-                        totals = per_zone.setdefault(zone, [0, 0.0, 0.0, 0.0, 0.0])
-                        for i, value in enumerate(values):
-                            totals[i] += value
+            per_zone = _sum_by_zone(self._days, horizon, zones)
+            approximate = _sum_by_zone(self._approximate, horizon, zones)
+            counts: dict[str, int] = {}
             for by_horizon in self._verdicts.values():
                 for verdict, count in by_horizon.get(str(horizon), {}).items():
-                    verdicts[verdict] = verdicts.get(verdict, 0) + count
+                    counts[verdict] = counts.get(verdict, 0) + count
+            reasons = {reason: counts.get(f"incomplete:{reason}", 0) for reason in INCOMPLETE_REASONS}
             overall = [sum(values[i] for values in per_zone.values()) for i in range(5)]
-            result[horizon] = {**_operational_stats(overall, horizon, len(per_zone)), **verdicts,
-                               "zones": {zone: _operational_stats(values, horizon)
-                                         for zone, values in per_zone.items()}}
+            approx_overall = [sum(values[i] for values in approximate.values()) for i in range(5)]
+            result[horizon] = {
+                **_operational_stats(overall, horizon, len(per_zone)),
+                # Forecasts by how the relay compared with their plan.
+                "matched": counts.get("matched", 0), "approximate": counts.get("approximate", 0),
+                "counterfactual": counts.get("counterfactual", 0), "incomplete": sum(reasons.values()),
+                "incomplete_reasons": reasons,
+                "approximate_stats": _operational_stats(approx_overall, horizon, len(approximate)),
+                "zones": {zone: _operational_stats(values, horizon) for zone, values in per_zone.items()},
+            }
         return result
 
     def as_dict(self) -> dict[str, Any]:
-        return {"days": {day: {h: {zone: [v[0], *(round(x, 6) for x in v[1:])] for zone, v in by_zone.items()}
-                               for h, by_zone in by_horizon.items()} for day, by_horizon in self._days.items()},
-                "verdicts": self._verdicts}
+        def rounded(store):
+            return {day: {h: {zone: [v[0], *(round(x, 6) for x in v[1:])] for zone, v in by_zone.items()}
+                          for h, by_zone in by_horizon.items()} for day, by_horizon in store.items()}
+        return {"version": LEDGER_VERSION, "days": rounded(self._days),
+                "approximate": rounded(self._approximate), "verdicts": self._verdicts}
 
     def load(self, data: Any) -> None:
-        self._days, self._verdicts = {}, {}
-        if not isinstance(data, dict):
+        self._days, self._approximate, self._verdicts = {}, {}, {}
+        # 0.4.1 statistics used a looser plan criterion: they are not comparable.
+        if not isinstance(data, dict) or data.get("version") != LEDGER_VERSION:
             return
-        horizons = {str(h) for h in HORIZONS}
-        days = data.get("days") if isinstance(data.get("days"), dict) else {}
-        for day, by_horizon in days.items():
-            if not _valid_day(day) or not isinstance(by_horizon, dict):
-                continue
-            for horizon, by_zone in by_horizon.items():
-                if horizon not in horizons or not isinstance(by_zone, dict):
-                    continue
-                for zone, values in by_zone.items():
-                    if (isinstance(zone, str) and isinstance(values, list) and len(values) == 5
-                            and type(values[0]) is int and values[0] >= 0
-                            and all(type(v) in (int, float) and isfinite(v) for v in values[1:])
-                            and min(values[2:]) >= 0):
-                        self._days.setdefault(day, {}).setdefault(horizon, {})[zone] = [
-                            values[0], *(float(v) for v in values[1:])]
+        for name, store in (("days", self._days), ("approximate", self._approximate)):
+            for day, horizon, zone, values in _entries(data.get(name)):
+                if (isinstance(values, list) and len(values) == 5 and type(values[0]) is int and values[0] >= 0
+                        and all(type(v) in (int, float) and isfinite(v) for v in values[1:])
+                        and min(values[2:]) >= 0):
+                    store.setdefault(day, {}).setdefault(horizon, {})[zone] = [
+                        values[0], *(float(v) for v in values[1:])]
+        known = {"matched", "approximate", "counterfactual", *(f"incomplete:{r}" for r in INCOMPLETE_REASONS)}
         verdicts = data.get("verdicts") if isinstance(data.get("verdicts"), dict) else {}
+        horizons = {str(h) for h in HORIZONS}
         for day, by_horizon in verdicts.items():
             if not _valid_day(day) or not isinstance(by_horizon, dict):
                 continue
             for horizon, counts in by_horizon.items():
                 if horizon in horizons and isinstance(counts, dict):
-                    clean = {k: v for k, v in counts.items() if k in VERDICTS and type(v) is int and v >= 0}
+                    clean = {k: v for k, v in counts.items() if k in known and type(v) is int and v >= 0}
                     if clean:
                         self._verdicts.setdefault(day, {})[horizon] = clean
 
 
+def _sum_by_zone(store: dict[str, dict[str, dict[str, list[float]]]], horizon: int,
+                 zones: list[str]) -> dict[str, list[float]]:
+    totals: dict[str, list[float]] = {}
+    for by_horizon in store.values():
+        for zone, values in by_horizon.get(str(horizon), {}).items():
+            if zone in zones:
+                bucket = totals.setdefault(zone, [0, 0.0, 0.0, 0.0, 0.0])
+                for i, value in enumerate(values):
+                    bucket[i] += value
+    return totals
+
+
 def _valid_day(day: Any) -> bool:
     try:
         datetime.strptime(day, "%Y-%m-%d")

custom_components/heating_mpc/sensor.py

@@ -229,7 +229,8 @@ class MPCOperationalForecastError(CoordinatorEntity, SensorEntity):
             horizons[f"{horizon}_min"] = {**{k: v for k, v in stats.items() if k != "zones"},
                                           "zones": {names.get(z, z): v for z, v in stats.get("zones", {}).items()}}
         return {
-            "kind": "forecast accuracy: published forecasts, scored only where the relay followed the plan",
+            "kind": ("forecast accuracy: published forecasts, scored only where the relay matched the plan "
+                     "(at most 2 minutes in another state); approximate matches (up to 5 minutes) apart"),
             "horizons": horizons,
             "window_days": WINDOW_DAYS,
             "unit_of_attributes": "°C",