Separate model accuracy from forecast accuracy

The Forecast Error sensors replayed the model with the relay, electrical heat
and window history that actually followed, so they measured the model, not a
forecast anyone could have made. They are renamed Model Error (unique IDs
unchanged) and gain RMSE, effective sample count and a quality label; their
state stays unknown until there are at least 5 roughly independent spans.

A new Forecast Error sensor scores forecasts as published, with the plan and
assumptions of the moment they were made. A forecast is scored only when the
relay followed its plan; otherwise it is counted as counterfactual, not as
error. Statistics are saved with the model and dropped with a zone's
recalibration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

custom_components/heating_mpc/coordinator.py

@@ -2,6 +2,8 @@
 from __future__ import annotations
 
 import asyncio
+import hashlib
+import json
 import logging
 from collections import deque
 from datetime import datetime, timedelta, timezone
@@ -23,7 +25,7 @@ from .const import (
 )
 from .compatibility import FINGERPRINT_VERSION, baseline_fingerprints, changed, reason, zone_fingerprints
 from .data_quality import ReportAgeHistory, freshness, inputs_differ
-from .evaluation import ForecastEvaluator, model_snapshot
+from .evaluation import ForecastEvaluator, OperationalForecastLedger, model_snapshot
 from .heat_sources import SUBENTRY_HEAT_SOURCE, HeatSource, HeatSourceTracker, build_sources, normalize_power
 from .learning import FeatureLearner
 from .physics import (
@@ -129,6 +131,9 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
         self._retired: dict[str, dict[str, Any]] = {}
         self._report_ages = ReportAgeHistory()
         self._evaluator = ForecastEvaluator(LAG_LEARN_MAX_UNCERTAINTY)
+        self._forecasts = OperationalForecastLedger()
+        # Seconds the relay was on during the last interval (None if unknown), and its length.
+        self._relay_interval: tuple[float | None, float | None] = (None, None)
         # How each poll was handled since start, to explain slow learning.
         self._sample_results: dict[str, int] = {}
         self._store = Store(hass, 1, f"{DOMAIN}.{entry.entry_id}")
@@ -167,6 +172,7 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             _LOGGER.warning("Heating MPC heater changed; starting a fresh thermal model")
             return
         self._evaluator.load(stored.get("evaluation"))
+        self._forecasts.load(stored.get("operational_evaluation"))
         changes = stored.get("model_changes")
         self._model_changes.extend(c for c in (changes if isinstance(changes, list) else []) if isinstance(c, dict))
         self._heat_sources = HeatSourceTracker(self.sources, stored.get("heat_sources"), self._baseline_prints)
@@ -227,6 +233,7 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
                 # The zone temperature is defined differently: re-learn it from a warm start.
                 self.learners[room].recalibrate()
                 self._evaluator.forget(room)
+                self._forecasts.forget(room)
                 resets.append("base")
             for layer in (self.extended.get(room), self.ventilation.get(room)):
                 if layer is None:
@@ -265,6 +272,7 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             "ventilation_models": {key: model.as_dict() for key, model in self.ventilation.items()},
             "ventilation": self._ventilation.as_dict(),
             "evaluation": self._evaluator.as_dict(),
+            "operational_evaluation": self._forecasts.as_dict(),
             "fingerprint_version": FINGERPRINT_VERSION,
             "fingerprints": self._fingerprints,
             "model_changes": list(self._model_changes),
@@ -316,6 +324,7 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
         tracker = self._lag
         steady = False
         lag = uncertainty = None
+        self._relay_interval = (None, None)  # unknown unless the relay history is known below
         if tracker is not None and now > tracker["time"]:
             start = tracker["time"]
             changes = [(when, state) for when, state in self._relay_events if start < when <= now]
@@ -326,10 +335,13 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             segments = None if self._relay_events_dropped else relay_segments(
                 tracker["heater_on"], start.timestamp(),
                 [(when.timestamp(), state) for when, state in changes], now.timestamp())
+            self._relay_interval = (None, (now - start).total_seconds())
             if segments is not None and segments[-1][0] == heater_on:
                 lag = tracker["lag"]
                 for command, seconds in segments:
                     lag = advance_lag(lag, command, seconds)
+                self._relay_interval = (sum(sec for command, sec in segments if command),
+                                        (now - start).total_seconds())
                 # The lag update is a contraction, so an initial error decays the same way.
                 uncertainty = tracker["uncertainty"] * exp(
                     -(now - start).total_seconds() / LAG_TAU_SECONDS)
@@ -343,6 +355,10 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
         self._relay_events_dropped = False
         return steady
 
+    def _revision(self) -> str:
+        """Short identity of the model configuration a forecast was made with."""
+        return hashlib.sha1(json.dumps(self._fingerprints, sort_keys=True).encode()).hexdigest()[:10]
+
     def _superseded(self, status: dict[str, Any]) -> dict[str, Any]:
         """Result of an update that a newer one (or unloading) overtook: keep what is shown."""
         current = getattr(self, "data", None)
@@ -612,6 +628,12 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             self._lag = None
             self._relay_events.clear()
             self._relay_events_dropped = False
+            self._relay_interval = (None, None)
+        # Published forecasts that are due are scored against these readings; nothing
+        # here is fed back into what those forecasts assumed.
+        self._forecasts.observe(now.timestamp(), {k: v for k, v in temps.items() if v is not None},
+                                *self._relay_interval)
+        status["operational_forecast"] = self._forecasts.summary(self.rooms)
         # Rooms without a usable reading are left out and the others carry on;
         # only the shared inputs, or losing every room, pause the integration.
         vals: dict[str, float] = {room: float(t) for room, t in temps.items() if t is not None}
@@ -782,12 +804,21 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             _LOGGER.warning("MPC prediction failed: %s", err)
             status["status"] = "prediction_error"
             return status
+        self._forecasts.record(now.timestamp(), vals, output["trajectory_15min"], output["plan_15min"],
+                               self._revision(), {
+            "strategy": "mpc_plan", "outside": outside, "wind": weather.get("wind"),
+            "internal_half_life_minutes": float(self.config.get(CONF_INTERNAL_HALF_LIFE, 30)),
+            "window_open_minutes": float(self.config[CONF_WINDOW_OPEN_MINUTES]),
+            "internal_kw": {room: internal[room] for room in vals},
+            "windows_open": [w.name for w in self.windows
+                             if self._ventilation.window_state.get(w.key, {}).get("open")]})
         status.update({
             "status": "ready", "proposal": output["proposal"],
             "prediction_60m": output["predicted_60m"],
             "prediction_end": output["predicted_end"],
             "plan_15min": output["plan_15min"], "objective": output["cost"],
             "prediction_trajectory": output["trajectory_15min"],
-            "model_quality_note": "5-minute in-sample MAE; not a validated 60-minute forecast error",
+            "model_quality_note": ("mae_5m is the in-sample 5-minute error; Model Error sensors replay the "
+                                   "model with actual inputs; Forecast Error scores published forecasts"),
         })
         return status