Keep spans with unknown inputs out of Model Error

Reproduced: a window unknown for a whole 15-minute span was replayed as
closed and scored as a Model Error sample (MAE 0.76 °C on one sample).

- A zone's replay is complete only if every step's model inputs were
  observed: window openings (and a confirmed recovery), the wind, direction
  and other zones' windows that an open window's terms use, and, for the
  extended layer, electrical heat and neighbours, whose own replay must be
  complete too.
- Incomplete spans are counted, not scored; the base model's error on them
  is kept separately (it needs only the relay and outdoor temperature).
  Spans that could not be replayed (uncertain radiator lag, no reading at
  the end) are counted as skipped. Model Error sensors show incomplete,
  skipped and incomplete_base_model_mae; stored data gains an "excluded"
  section and older data loads unchanged.
- Window inputs no longer stand in for unknown values: the facing term is
  unknown while a window is open and the wind direction is not, and the
  cross-ventilation input is unknown while another zone's window is. With
  every window closed, opening terms are 0 and need no weather input.
  Learning refuses such samples; forecasts still assume calm air.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

custom_components/heating_mpc/evaluation.py

@@ -34,7 +34,7 @@ from math import isfinite, sqrt
 from typing import Any
 
 from .physics import Learner
-from .windows import ventilation_vector_for_prediction
+from .windows import ventilation_known, ventilation_vector_for_prediction
 from .zone_model import feature_vector
 
 HORIZONS = (15, 30, 60, 120)  # minutes
@@ -97,19 +97,50 @@ def snapshot_delta(snapshot: dict[str, Any], temp: float, inputs: dict[str, Any]
     return main
 
 
-def replay_zones(origin: dict[str, Any], path: list[dict[str, Any]]) -> dict[str, float]:
-    """All zones advanced together with the origin's models and the actual inputs."""
+def inputs_complete(snapshot: dict[str, Any], temp: float, inputs: dict[str, Any],
+                    incomplete: set[str]) -> bool:
+    """Whether a step of a zone's model used only observed inputs.
+
+    Not when a window state, or a weather input an open window's terms need, was
+    unknown; nor when the extended layer is in use and its electrical heat or a
+    neighbour is unknown, or that neighbour's own replay is already incomplete.
+    The replay itself still falls back as forecasts do; the result is just not a
+    measurement of the model.
+    """
+    vent_features = snapshot["ventilation"][0] if snapshot.get("ventilation") else ()
+    if not ventilation_known(vent_features, inputs.get("ventilation")):
+        return False
+    if snapshot["uses_extended"] and snapshot["extended"]:
+        features = snapshot["extended"][0]
+        if feature_vector(features, temp, inputs) is None:
+            return False
+        if any(f.startswith("couple:") and f.split(":", 1)[1] in incomplete for f in features):
+            return False
+    return True
+
+
+def replay_zones(origin: dict[str, Any], path: list[dict[str, Any]]) -> tuple[dict[str, float], set[str]]:
+    """All zones advanced together with the origin's models and the actual inputs.
+
+    Returns the replayed temperatures and the zones whose replay needed an input
+    that was not observed at some step (see inputs_complete).
+    """
     state = dict(origin["temps"])
+    incomplete: set[str] = set()
     for before, poll in zip(path, path[1:]):
         seconds = (poll["time"] - before["time"]).total_seconds()
         new = {}
+        unknown = set(incomplete)
         for zone, temp in state.items():
             actual = poll["inputs"].get(zone, {})
             inputs = {"outside": origin["outside"], "lag": poll["lag"], "temps": state,
                       "internal": actual.get("internal"), "ventilation": actual.get("ventilation")}
-            new[zone] = temp + snapshot_delta(origin["models"][zone], temp, inputs) * seconds / STEP_SECONDS
-        state = new
-    return state
+            snapshot = origin["models"][zone]
+            if not inputs_complete(snapshot, temp, inputs, incomplete):
+                unknown.add(zone)
+            new[zone] = temp + snapshot_delta(snapshot, temp, inputs) * seconds / STEP_SECONDS
+        state, incomplete = new, unknown
+    return state, incomplete
 
 
 class ForecastEvaluator:
@@ -121,6 +152,9 @@ class ForecastEvaluator:
         # UTC day -> horizon -> room -> [count, sum error, sum |error|, sum |persistence error|,
         #   sum |base model error|, sum error^2, count with squares (0 for 0.3/0.4 data)]
         self._days: dict[str, dict[str, dict[str, list[float]]]] = {}
+        # UTC day -> horizon -> room -> [incomplete spans, skipped spans, sum |base model error|
+        #   over the incomplete ones]. Neither kind enters the statistics above.
+        self._excluded: dict[str, dict[str, dict[str, list[float]]]] = {}
 
     def reset_window(self) -> None:
         """Inputs were missing: no forecast may be scored across this poll."""
@@ -147,6 +181,11 @@ class ForecastEvaluator:
         self._score()
         self._prune(time)
 
+    def _exclude(self, day: str, horizon: int, room: str, kind: int, base_error: float = 0.0) -> None:
+        bucket = self._excluded.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(room, [0, 0, 0.0])
+        bucket[kind] += 1
+        bucket[2] += abs(base_error)
+
     def _score(self) -> None:
         history = list(self._history)
         latest = history[-1]
@@ -156,19 +195,27 @@ class ForecastEvaluator:
             if len(history) < steps + 1:
                 continue
             path = history[-(steps + 1):]
+            origin = path[0]
             # The actual heat input must be known along the way.
             if any(poll["lag_uncertainty"] > self._max_lag_uncertainty for poll in path[1:]):
+                for room in origin["temps"]:
+                    self._exclude(day, horizon, room, SKIPPED)
                 continue
-            origin = path[0]
             inputs = [(poll["lag"], (poll["time"] - before["time"]).total_seconds())
                       for before, poll in zip(path, path[1:])]
-            full = replay_zones(origin, path)
-            for room, actual in latest["temps"].items():
-                start, theta = origin["temps"].get(room), origin["theta"].get(room)
-                if start is None or theta is None:
+            full, incomplete = replay_zones(origin, path)
+            for room, start in origin["temps"].items():
+                actual, theta = latest["temps"].get(room), origin["theta"].get(room)
+                if actual is None or theta is None:
+                    self._exclude(day, horizon, room, SKIPPED)  # no reading at the end
                     continue
                 error = full[room] - actual
                 base_error = replay(start, origin["outside"], theta, inputs) - actual
+                if room in incomplete:
+                    # The base model needs only the relay and outdoor temperature: its
+                    # error is still a measurement, kept apart from the main figures.
+                    self._exclude(day, horizon, room, INCOMPLETE, base_error)
+                    continue
                 bucket = self._days.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
                     room, [0, 0.0, 0.0, 0.0, 0.0, 0.0, 0])
                 bucket[0] += 1
@@ -181,8 +228,9 @@ class ForecastEvaluator:
 
     def _prune(self, time: datetime) -> None:
         oldest = (time.date() - timedelta(days=WINDOW_DAYS - 1)).isoformat()
-        for day in [day for day in self._days if day < oldest]:
-            del self._days[day]
+        for store in (self._days, self._excluded):
+            for day in [day for day in store if day < oldest]:
+                del store[day]
 
     def summary(self, rooms: list[str]) -> dict[int, dict[str, Any]]:
         """Error statistics per horizon, overall and per configured room."""
@@ -196,15 +244,27 @@ class ForecastEvaluator:
                         for i, value in enumerate(values):
                             totals[i] += value
             overall = [sum(values[i] for values in per_room.values()) for i in range(7)]
-            result[horizon] = {**_stats(overall, horizon, len(per_room)),
-                               "rooms": {room: _stats(values, horizon) for room, values in per_room.items()}}
+            excluded: dict[str, list[float]] = {}
+            for day in self._excluded.values():
+                for room, values in day.get(str(horizon), {}).items():
+                    if room in rooms:
+                        totals = excluded.setdefault(room, [0, 0, 0.0])
+                        for i, value in enumerate(values):
+                            totals[i] += value
+            all_excluded = [sum(values[i] for values in excluded.values()) for i in range(3)]
+            result[horizon] = {**_stats(overall, horizon, len(per_room)), **_excluded_stats(all_excluded),
+                               "rooms": {room: {**_stats(per_room.get(room, [0, 0.0, 0.0, 0.0, 0.0, 0.0, 0]),
+                                                         horizon),
+                                                **_excluded_stats(excluded.get(room, [0, 0, 0.0]))}
+                                         for room in rooms if room in per_room or room in excluded}}
         return result
 
     def forget(self, room: str) -> None:
         """Drop a zone's statistics, for example when its temperature is measured differently."""
-        for by_horizon in self._days.values():
-            for by_room in by_horizon.values():
-                by_room.pop(room, None)
+        for store in (self._days, self._excluded):
+            for by_horizon in store.values():
+                for by_room in by_horizon.values():
+                    by_room.pop(room, None)
         for poll in self._history:
             poll["temps"].pop(room, None)
 
@@ -213,11 +273,20 @@ class ForecastEvaluator:
         return {"days": {day: {horizon: {room: [v[0], *(round(x, 6) for x in v[1:6]), v[6]]
                                          for room, v in by_room.items()}
                                for horizon, by_room in by_horizon.items()}
-                         for day, by_horizon in self._days.items()}}
+                         for day, by_horizon in self._days.items()},
+                "excluded": {day: {horizon: {room: [v[0], v[1], round(v[2], 6)] for room, v in by_room.items()}
+                                   for horizon, by_room in by_horizon.items()}
+                             for day, by_horizon in self._excluded.items()}}
 
     def load(self, data: Any) -> None:
         """Restore stored statistics, keeping only well-formed entries."""
-        self._days = {}
+        self._days, self._excluded = {}, {}
+        excluded = data.get("excluded") if isinstance(data, dict) else None
+        for day, horizon, room, values in _entries(excluded):
+            if (isinstance(values, list) and len(values) == 3 and all(type(v) is int and v >= 0 for v in values[:2])
+                    and type(values[2]) in (int, float) and isfinite(values[2]) and values[2] >= 0):
+                self._excluded.setdefault(day, {}).setdefault(horizon, {})[room] = [
+                    values[0], values[1], float(values[2])]
         days = data.get("days") if isinstance(data, dict) else None
         if not isinstance(days, dict):
             return
@@ -243,6 +312,30 @@ class ForecastEvaluator:
                             *squares]
 
 
+INCOMPLETE, SKIPPED = 0, 1  # columns of the excluded-span counts
+
+
+def _entries(days: Any):
+    """(day, horizon, zone, values) of stored per-day statistics with a valid shape."""
+    if not isinstance(days, dict):
+        return
+    horizons = {str(horizon) for horizon in HORIZONS}
+    for day, by_horizon in days.items():
+        if not _valid_day(day) or not isinstance(by_horizon, dict):
+            continue
+        for horizon, by_zone in by_horizon.items():
+            if horizon in horizons and isinstance(by_zone, dict):
+                for zone, values in by_zone.items():
+                    if isinstance(zone, str):
+                        yield day, horizon, zone, values
+
+
+def _excluded_stats(values: list[float]) -> dict[str, Any]:
+    incomplete, skipped, base = values
+    return {"incomplete": int(incomplete), "skipped": int(skipped),
+            "incomplete_base_model_mae": round(base / incomplete, 3) if incomplete else None}
+
+
 def _valid_bucket(room: Any, values: Any) -> bool:
     if not isinstance(room, str) or not isinstance(values, list) or len(values) not in (4, 5, 7):
         return False