Score forecasts against reality; expose forecast trajectories

There was no evidence of how accurate the thermal model is: the only
number was an in-sample 5-minute error.

At every poll, record each room's reading, the outdoor temperature, the
radiator-lag estimate and the model parameters. 15, 30, 60 and 120
minutes later, replay the room from that poll with the parameters it
had then (out of sample), the outdoor temperature held constant and the
radiator lag that actually followed from the relay's real history, and
compare with the reading now. A no-change forecast is scored alongside.
Only spans with regular polls and a well-known lag are scored. 7 days
of per-room statistics are saved with the model (optional "evaluation"
payload key; about 18 KB at most).

Four new sensors, Heating MPC Forecast Error 15/30/60/120 min, show the
mean absolute error with bias, persistence error, skill and per-room
detail. On the synthetic lagged plant the 60-minute error is 0.03 °C
against 0.39 °C for persistence.

Room forecast entities gain forecast_15min and plan_15min attributes
and are labelled counterfactual. Volatile attributes are unrecorded.
The model is now saved after scoring, so saved statistics are current.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

custom_components/heating_mpc/coordinator.py

@@ -20,6 +20,7 @@ from .const import (
     CONF_BAND, DEFAULTS,
 )
 from .data_quality import ReportAgeHistory, freshness
+from .evaluation import ForecastEvaluator
 from .physics import (
     LAG_LEARN_MAX_UNCERTAINTY, LAG_TAU_SECONDS, Learner, advance_lag,
     lag_from_time_in_state, optimize, relay_segments,
@@ -45,6 +46,7 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
         self.learners: dict[str, Learner] = {room: Learner() for room in self.rooms}
         self._retired: dict[str, dict[str, Any]] = {}
         self._report_ages = ReportAgeHistory()
+        self._evaluator = ForecastEvaluator(LAG_LEARN_MAX_UNCERTAINTY)
         # How each poll was handled since start, to explain slow learning.
         self._sample_results: dict[str, int] = {}
         self._store = Store(hass, 1, f"{DOMAIN}.{entry.entry_id}")
@@ -68,6 +70,7 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
         if stored.get("heater_entity", self.config[CONF_HEATER]) != self.config[CONF_HEATER]:
             _LOGGER.warning("Heating MPC heater changed; starting a fresh thermal model")
             return
+        self._evaluator.load(stored.get("evaluation"))
         # Models carry room identity, so changing the room list is safe. Models of
         # rooms no longer configured are retained, most recently removed last.
         for key in ("retired_learners", "learners"):  # current learners take precedence
@@ -255,10 +258,13 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
         vals: dict[str, float] = {room: float(t) for room, t in temps.items() if t is not None}
         if not heater_valid or climate is None or outside is None or not vals:
             self._previous = None
+            self._evaluator.reset_window()
+            status["forecast_accuracy"] = self._evaluator.summary(self.rooms)
             return status
 
         # A room learns only across a poll pair where it was usable at both ends.
         pairs = [room for room in vals if self._previous and room in self._previous["temps"]]
+        learned = False
         if not pairs:
             status["sample_result"] = "first_sample"
         else:
@@ -282,16 +288,7 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
                 ]
                 status["sample_result"] = "learned" if all(accepted) else "partly_rejected"
                 self._update_counts()
-                # Persist at each sample so restart does not erase hours of training.
-                await self._store.async_save({
-                    "schema_version": 2,
-                    "learners": {room: learner.as_dict() for room, learner in self.learners.items()},
-                    # Optional; older 0.2.x readers ignore it and keep working.
-                    "retired_learners": self._retired,
-                    "on_samples": self._on_samples, "off_samples": self._off_samples,
-                    "heater_entity": self.config[CONF_HEATER],
-                    "outside_source": active_sources["temperature"],
-                })
+                learned = True
 
         result = status["sample_result"]
         self._sample_results[result] = self._sample_results.get(result, 0) + 1
@@ -300,6 +297,22 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             "time": now, "temps": vals, "outside": outside, "heater_on": heater_on,
             "outside_source": active_sources["temperature"],
         }
+        # Parameters after this poll's learning: forecasts from here use only data so far.
+        self._evaluator.observe(now, vals, outside, self._lag["lag"], self._lag["uncertainty"],
+                                {room: tuple(self.learners[room].theta) for room in vals})
+        status["forecast_accuracy"] = self._evaluator.summary(self.rooms)
+        if learned:
+            # Persist at each sample so restart does not erase hours of training.
+            await self._store.async_save({
+                "schema_version": 2,
+                "learners": {room: learner.as_dict() for room, learner in self.learners.items()},
+                # Optional keys; older 0.2.x readers ignore them and keep working.
+                "retired_learners": self._retired,
+                "evaluation": self._evaluator.as_dict(),
+                "on_samples": self._on_samples, "off_samples": self._off_samples,
+                "heater_entity": self.config[CONF_HEATER],
+                "outside_source": active_sources["temperature"],
+            })
         weights = self.config.get(CONF_WEIGHTS, {})
         try:
             weights = {r: float(weights.get(r, 1.0)) for r in self.rooms}
@@ -381,6 +394,7 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             "prediction_60m": output["predicted_60m"],
             "prediction_end": output["predicted_end"],
             "plan_15min": output["plan_15min"], "objective": output["cost"],
+            "prediction_trajectory": output["trajectory_15min"],
             "model_quality_note": "5-minute in-sample MAE; not a validated 60-minute forecast error",
         })
         return status