Separate model accuracy from forecast accuracy

The Forecast Error sensors replayed the model with the relay, electrical heat
and window history that actually followed, so they measured the model, not a
forecast anyone could have made. They are renamed Model Error (unique IDs
unchanged) and gain RMSE, effective sample count and a quality label; their
state stays unknown until there are at least 5 roughly independent spans.

A new Forecast Error sensor scores forecasts as published, with the plan and
assumptions of the moment they were made. A forecast is scored only when the
relay followed its plan; otherwise it is counted as counterfactual, not as
error. Statistics are saved with the model and dropped with a zone's
recalibration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_coordinator.py

@@ -1117,8 +1117,8 @@ class BoundaryTests(unittest.IsolatedAsyncioTestCase):
         vent.count = 80
         vent.theta[vent.features.index('vent_natural')] = 0.01
         vent.theta[vent.features.index('vent_wind_dir')] = 0.0005
-        c._evaluator._days = {'2026-10-27': {'60': {'living': [10, 0.1, 0.2, 0.3, 0.4],
-                                                    'bed': [10, 0.1, 0.2, 0.3, 0.4]}}}
+        c._evaluator._days = {'2026-10-27': {'60': {'living': [10, 0.1, 0.2, 0.3, 0.4, 0.05, 10],
+                                                    'bed': [10, 0.1, 0.2, 0.3, 0.4, 0.05, 10]}}}
         c._heat_sources._baselines['ups'].samples.extend([200.0] * 24)
         return c._payload('sensor.outside')
 
@@ -1261,6 +1261,35 @@ class BoundaryTests(unittest.IsolatedAsyncioTestCase):
         self.assertEqual(newer['status'], 'ready')
         self.assertIs(result, newer)
 
+    async def test_published_forecasts_are_scored_only_when_the_relay_followed_the_plan(self):
+        # The relay stays off. A warm room's plan says off, so its forecast is scored;
+        # a cold room's plan says heat, so its forecast is counterfactual, never an error.
+        for room, follows in (('23.5', True), ('18', False)):
+            with self.subTest(room=room):
+                self.setUp()
+                self.ready_office()
+                self.set_state('sensor.room', room, unit_of_measurement='°C')
+                first = await self.sample()
+                self.assertEqual(first['status'], 'ready')
+                self.assertEqual(first['plan_15min'][0], 0 if follows else 1)
+                for _ in range(3):
+                    self.fresh('sensor.room', 'sensor.outside', 'sensor.ups_power')
+                    data = await self.sample()
+                fifteen = data['operational_forecast'][15]
+                if follows:
+                    self.assertEqual((fifteen['samples'], fifteen['counterfactual']), (1, 0))
+                    predicted = first['prediction_trajectory']['office'][0]
+                    self.assertAlmostEqual(fifteen['mae'], abs(predicted - 23.5), places=3)
+                    self.assertEqual(fifteen['persistence_mae'], 0.0)
+                    self.assertEqual(fifteen['quality'], 'insufficient')
+                else:
+                    self.assertEqual((fifteen['samples'], fifteen['counterfactual'], fifteen['mae']), (0, 1, None))
+                # Longer horizons are not due yet.
+                self.assertEqual(data['operational_forecast'][30]['samples'], 0)
+                stored = self.c._payload('sensor.outside')['operational_evaluation']
+                self.assertEqual(sum(sum(v.get('15', {}).values()) for v in stored['verdicts'].values()),
+                                 0 if follows else 1)
+
     async def test_unloading_during_save_or_optimization_publishes_nothing_new(self):
         self.ready_office()
         await self.sample()                       # a first sample to learn from