Separate model accuracy from forecast accuracy

The Forecast Error sensors replayed the model with the relay, electrical heat
and window history that actually followed, so they measured the model, not a
forecast anyone could have made. They are renamed Model Error (unique IDs
unchanged) and gain RMSE, effective sample count and a quality label; their
state stays unknown until there are at least 5 roughly independent spans.

A new Forecast Error sensor scores forecasts as published, with the plan and
assumptions of the moment they were made. A forecast is scored only when the
relay followed its plan; otherwise it is counted as counterfactual, not as
error. Statistics are saved with the model and dropped with a zone's
recalibration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

custom_components/heating_mpc/sensor.py

@@ -16,6 +16,7 @@ async def async_setup_entry(hass, entry, async_add_entities) -> None:
     entities = [MPCStatus(coordinator, entry.entry_id), MPCWeightedTemp(coordinator, entry.entry_id), MPCOutdoorTemperature(coordinator, entry.entry_id),
                 MPCLearningProgress(coordinator, entry.entry_id), MPCLastSample(coordinator, entry.entry_id)]
     entities += [MPCForecastError(coordinator, entry.entry_id, horizon) for horizon in HORIZONS]
+    entities.append(MPCOperationalForecastError(coordinator, entry.entry_id))
     # Zone forecasts belong to their zone subentry, so deleting a zone removes them.
     linked = {}
     for zone in coordinator.zones.values():
@@ -52,7 +53,7 @@ class MPCStatus(CoordinatorEntity, SensorEntity):
         # Forecast details live on their own entities.
         return {key: value for key, value in data.items() if key not in (
             "status", "prediction_60m", "prediction_end", "prediction_trajectory", "forecast_accuracy",
-            "zone_details")}
+            "zone_details", "operational_forecast")}
 
 
 class MPCWeightedTemp(CoordinatorEntity, SensorEntity):
@@ -133,7 +134,11 @@ class MPCOutdoorTemperature(CoordinatorEntity, SensorEntity):
 
 
 class MPCForecastError(CoordinatorEntity, SensorEntity):
-    """Mean absolute error of the thermal model at one horizon, replayed against reality."""
+    """Model accuracy at one horizon: the model replayed with the inputs that actually happened.
+
+    Not a forecast accuracy (a real forecast does not know the future relay, power or
+    windows); see MPCOperationalForecastError. The unique ID keeps its original name.
+    """
     _attr_has_entity_name = True
     _attr_device_class = SensorDeviceClass.TEMPERATURE_DELTA
     _attr_native_unit_of_measurement = UnitOfTemperature.CELSIUS
@@ -146,7 +151,7 @@ class MPCForecastError(CoordinatorEntity, SensorEntity):
         super().__init__(coordinator)
         self.horizon = horizon
         self._attr_unique_id = f"{entry_id}_forecast_error_{horizon}min"
-        self._attr_name = f"Heating MPC Forecast Error {horizon} min"
+        self._attr_name = f"Heating MPC Model Error {horizon} min"
 
     @property
     def _accuracy(self):
@@ -154,13 +159,20 @@ class MPCForecastError(CoordinatorEntity, SensorEntity):
 
     @property
     def native_value(self):
-        return self._accuracy.get("mae")
+        accuracy = self._accuracy
+        # No number from a handful of overlapping forecasts.
+        return accuracy.get("mae") if accuracy.get("quality", "insufficient") != "insufficient" else None
 
     @property
     def extra_state_attributes(self):
         accuracy = self._accuracy
         return {
+            "kind": "model accuracy: replayed with the actual relay, electrical heat and window history",
             "samples": accuracy.get("samples", 0),
+            "effective_samples": accuracy.get("effective_samples", 0),
+            "quality": accuracy.get("quality", "insufficient"),
+            "mae": accuracy.get("mae"),
+            "rmse": accuracy.get("rmse"),
             "bias": accuracy.get("bias"),
             "persistence_mae": accuracy.get("persistence_mae"),
             "skill": accuracy.get("skill"),
@@ -176,6 +188,47 @@ class MPCForecastError(CoordinatorEntity, SensorEntity):
         }
 
 
+class MPCOperationalForecastError(CoordinatorEntity, SensorEntity):
+    """Accuracy of forecasts as published, using only what was known when they were made.
+
+    A forecast assumes the MPC's plan; it is scored only when the relay actually
+    followed that plan, otherwise counted as counterfactual. State: the 60-minute
+    error; all horizons in the attributes.
+    """
+    _attr_has_entity_name = True
+    _attr_name = "Heating MPC Forecast Error"
+    _attr_device_class = SensorDeviceClass.TEMPERATURE_DELTA
+    _attr_native_unit_of_measurement = UnitOfTemperature.CELSIUS
+    _attr_state_class = SensorStateClass.MEASUREMENT
+    _attr_suggested_display_precision = 2
+    _attr_icon = "mdi:crystal-ball"
+    _unrecorded_attributes = frozenset({"horizons"})
+
+    def __init__(self, coordinator, entry_id):
+        super().__init__(coordinator)
+        self._attr_unique_id = f"{entry_id}_operational_forecast_error"
+
+    @property
+    def native_value(self):
+        hour = self.coordinator.data.get("operational_forecast", {}).get(60, {})
+        return hour.get("mae") if hour.get("quality", "insufficient") != "insufficient" else None
+
+    @property
+    def extra_state_attributes(self):
+        summary = self.coordinator.data.get("operational_forecast", {})
+        names = {key: zone.name for key, zone in self.coordinator.zones.items()}
+        horizons = {}
+        for horizon, stats in summary.items():
+            horizons[f"{horizon}_min"] = {**{k: v for k, v in stats.items() if k != "zones"},
+                                          "zones": {names.get(z, z): v for z, v in stats.get("zones", {}).items()}}
+        return {
+            "kind": "forecast accuracy: published forecasts, scored only where the relay followed the plan",
+            "horizons": horizons,
+            "window_days": WINDOW_DAYS,
+            "unit_of_attributes": "°C",
+        }
+
+
 class MPCLearningProgress(CoordinatorEntity, SensorEntity):
     """How close the least trained room is to being used for forecasts."""
     _attr_has_entity_name = True