Score the full model against the base model; compile the search step

Validation: the walk-forward evaluator now replays all zones together
with the full model (electrical heat, neighbour exchange, ventilation
and recovery) using the parameters frozen at the forecast start and the
electrical heat and window openings that actually happened, and scores
the base 3-parameter model on the same spans. Forecast Error sensors
add base_model_mae and improvement_vs_base (share of the base model's
error the new features remove; 0 = no gain, negative = worse). Saved
0.3 statistics load with their own error as the base column.

On synthetic apartments, 60-minute error full vs base: gaming office
0.031 vs 0.182 °C (83 % better), bedroom with ventilation 0.026 vs
0.218 °C (88 %), two coupled rooms 0.024 vs 0.037 °C (34 %; gone by
120 min, where the constant-outdoor assumption dominates). The same
ventilation week without window sensors shows no improvement, as it
should. These check the method, not real-world accuracy.

Performance: every term of a zone's 5-minute change is affine in the
zone temperatures and the radiator lag, and which layer applies depends
only on which inputs exist. compile_step_model reads the coefficients
off the real model per zone and step, so the plan search does a few
multiply-adds per step. It matches the direct model to 1e-12 and makes
a 180-minute, 5-zone search 4.5x faster (0.73 s here, close to the old
base-only search).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

custom_components/heating_mpc/evaluation.py

@@ -10,8 +10,11 @@ value, and the radiator lag that actually followed from the relay's real ON/OFF
 history. The replayed temperature is compared with the reading now.
 
 This scores the thermal model, not the MPC's proposed plan: Generic Thermostat
-controls the relay, so the plan is never executed. A persistence forecast (no
-change) is scored alongside as the baseline to beat.
+controls the relay, so the plan is never executed. Two baselines are scored on
+the same spans: persistence (no change) and the base 3-parameter model. The full
+model replays all zones together with the electrical heat, neighbour exchange and
+window openings that actually happened, so its improvement over the base model
+shows whether those features earn their keep.
 """
 from __future__ import annotations
 
@@ -21,6 +24,8 @@ from math import isfinite
 from typing import Any
 
 from .physics import Learner
+from .windows import ventilation_vector
+from .zone_model import feature_vector
 
 HORIZONS = (15, 30, 60, 120)  # minutes
 WINDOW_DAYS = 7
@@ -37,13 +42,60 @@ def replay(temp: float, outside: float, theta: tuple[float, float, float],
     return temp
 
 
+def _dot(theta: tuple[float, ...], x: list[float]) -> float:
+    return sum(t * v for t, v in zip(theta, x))
+
+
+def model_snapshot(model: Any) -> dict[str, Any]:
+    """Frozen parameters of a ZoneModel, so later learning never leaks into a replay."""
+    def layer(learner):
+        return (list(learner.features), tuple(learner.theta)) if learner is not None else None
+    return {"base": tuple(model.base.theta), "uses_extended": model.uses_extended,
+            "extended": layer(model.extended), "ventilation": layer(model.ventilation)}
+
+
+def snapshot_delta(snapshot: dict[str, Any], temp: float, inputs: dict[str, Any]) -> float:
+    """5-minute change predicted by a model snapshot (see zone_model.ZoneModel.delta)."""
+    main = None
+    if snapshot["uses_extended"] and snapshot["extended"]:
+        features, theta = snapshot["extended"]
+        x = feature_vector(features, temp, inputs)
+        main = _dot(theta, x) if x is not None else None
+    if main is None:
+        a, b, c = snapshot["base"]
+        main = a * (inputs["outside"] - temp) + b * inputs["lag"] + c
+    ventilation = inputs.get("ventilation")
+    if snapshot["ventilation"] and ventilation:
+        features, theta = snapshot["ventilation"]
+        x = ventilation_vector(features, temp, inputs["outside"], ventilation)
+        if x is not None:
+            main += _dot(theta, x)
+    return main
+
+
+def replay_zones(origin: dict[str, Any], path: list[dict[str, Any]]) -> dict[str, float]:
+    """All zones advanced together with the origin's models and the actual inputs."""
+    state = dict(origin["temps"])
+    for before, poll in zip(path, path[1:]):
+        seconds = (poll["time"] - before["time"]).total_seconds()
+        new = {}
+        for zone, temp in state.items():
+            actual = poll["inputs"].get(zone, {})
+            inputs = {"outside": origin["outside"], "lag": poll["lag"], "temps": state,
+                      "internal": actual.get("internal"), "ventilation": actual.get("ventilation")}
+            new[zone] = temp + snapshot_delta(origin["models"][zone], temp, inputs) * seconds / STEP_SECONDS
+        state = new
+    return state
+
+
 class ForecastEvaluator:
     """Score replayed forecasts per room and horizon over the last WINDOW_DAYS days."""
 
     def __init__(self, max_lag_uncertainty: float) -> None:
         self._max_lag_uncertainty = max_lag_uncertainty
         self._history: deque[dict[str, Any]] = deque(maxlen=HORIZONS[-1] * 60 // STEP_SECONDS + 1)
-        # UTC day -> horizon -> room -> [count, sum error, sum |error|, sum |persistence error|]
+        # UTC day -> horizon -> room ->
+        #   [count, sum error, sum |error|, sum |persistence error|, sum |base model error|]
         self._days: dict[str, dict[str, dict[str, list[float]]]] = {}
 
     def reset_window(self) -> None:
@@ -51,12 +103,23 @@ class ForecastEvaluator:
         self._history.clear()
 
     def observe(self, time: datetime, temps: dict[str, float], outside: float, lag: float,
-                lag_uncertainty: float, theta: dict[str, tuple[float, float, float]]) -> None:
-        """Record a poll, then score every forecast that has reached its horizon."""
+                lag_uncertainty: float, theta: dict[str, tuple[float, float, float]],
+                models: dict[str, dict[str, Any]] | None = None,
+                inputs: dict[str, dict[str, Any]] | None = None) -> None:
+        """Record a poll, then score every forecast that has reached its horizon.
+
+        `models`: model_snapshot per zone (base-only when omitted); `inputs`: this
+        poll's actual electrical heat and ventilation inputs per zone.
+        """
         if self._history and not 270 <= (time - self._history[-1]["time"]).total_seconds() <= 330:
             self._history.clear()  # irregular poll spacing: replays would be unreliable
+        models = models or {}
+        snapshots = {zone: models.get(zone) or {"base": tuple(theta[zone]), "uses_extended": False,
+                                               "extended": None, "ventilation": None}
+                     for zone in temps if zone in theta}
         self._history.append({"time": time, "temps": dict(temps), "outside": outside, "lag": lag,
-                              "lag_uncertainty": lag_uncertainty, "theta": dict(theta)})
+                              "lag_uncertainty": lag_uncertainty, "theta": dict(theta),
+                              "models": snapshots, "inputs": dict(inputs or {})})
         self._score()
         self._prune(time)
 
@@ -75,17 +138,20 @@ class ForecastEvaluator:
             origin = path[0]
             inputs = [(poll["lag"], (poll["time"] - before["time"]).total_seconds())
                       for before, poll in zip(path, path[1:])]
+            full = replay_zones(origin, path)
             for room, actual in latest["temps"].items():
                 start, theta = origin["temps"].get(room), origin["theta"].get(room)
                 if start is None or theta is None:
                     continue
-                error = replay(start, origin["outside"], theta, inputs) - actual
+                error = full[room] - actual
+                base_error = replay(start, origin["outside"], theta, inputs) - actual
                 bucket = self._days.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
-                    room, [0, 0.0, 0.0, 0.0])
+                    room, [0, 0.0, 0.0, 0.0, 0.0])
                 bucket[0] += 1
                 bucket[1] += error
                 bucket[2] += abs(error)
                 bucket[3] += abs(start - actual)
+                bucket[4] += abs(base_error)
 
     def _prune(self, time: datetime) -> None:
         oldest = (time.date() - timedelta(days=WINDOW_DAYS - 1)).isoformat()
@@ -100,10 +166,10 @@ class ForecastEvaluator:
             for day in self._days.values():
                 for room, values in day.get(str(horizon), {}).items():
                     if room in rooms:
-                        totals = per_room.setdefault(room, [0, 0.0, 0.0, 0.0])
+                        totals = per_room.setdefault(room, [0, 0.0, 0.0, 0.0, 0.0])
                         for i, value in enumerate(values):
                             totals[i] += value
-            overall = [sum(values[i] for values in per_room.values()) for i in range(4)]
+            overall = [sum(values[i] for values in per_room.values()) for i in range(5)]
             result[horizon] = {**_stats(overall),
                                "rooms": {room: _stats(values) for room, values in per_room.items()}}
         return result
@@ -134,23 +200,26 @@ class ForecastEvaluator:
                     continue
                 for room, values in by_room.items():
                     if _valid_bucket(room, values):
+                        # 0.3 buckets had no base column: the model then was the base model.
+                        base = values[4] if len(values) == 5 else values[2]
                         self._days.setdefault(day, {}).setdefault(horizon, {})[room] = [
-                            values[0], float(values[1]), float(values[2]), float(values[3])]
+                            values[0], float(values[1]), float(values[2]), float(values[3]), float(base)]
 
 
 def _valid_bucket(room: Any, values: Any) -> bool:
-    if not isinstance(room, str) or not isinstance(values, list) or len(values) != 4:
+    if not isinstance(room, str) or not isinstance(values, list) or len(values) not in (4, 5):
         return False
     count, *sums = values
     return (type(count) is int and count >= 0
             and all(type(v) in (int, float) and isfinite(v) for v in sums)
-            and sums[1] >= 0 and sums[2] >= 0)
+            and all(v >= 0 for v in sums[1:]))
 
 
 def _stats(values: list[float]) -> dict[str, Any]:
-    count, error, absolute, persistence = values
+    count, error, absolute, persistence, base = values
     if not count:
-        return {"samples": 0, "mae": None, "bias": None, "persistence_mae": None, "skill": None}
+        return {"samples": 0, "mae": None, "bias": None, "persistence_mae": None, "skill": None,
+                "base_model_mae": None, "improvement_vs_base": None}
     return {
         "samples": int(count),
         "mae": round(absolute / count, 3),
@@ -158,4 +227,8 @@ def _stats(values: list[float]) -> dict[str, Any]:
         "persistence_mae": round(persistence / count, 3),
         # Share of the persistence error removed by the model (1 = perfect, < 0 = worse).
         "skill": round(1 - absolute / persistence, 3) if persistence > 0 else None,
+        "base_model_mae": round(base / count, 3),
+        # Share of the base model's error removed by electrical heat, neighbours and
+        # ventilation (0 = no gain, < 0 = the extra features make it worse).
+        "improvement_vs_base": round(1 - absolute / base, 3) if base > 0 else None,
     }