Score a forecast only when the relay matched its plan in time, not just in share

Reproduced: a forecast planning 15 minutes of heat was scored although the
relay was on for 12 (the 0.4.1 rule allowed 20 % per 15-minute block), and
the same on-time placed elsewhere in a block passed as well.

- The relay's on-time per poll interval, integrated at its own switch
  times, is compared with the plan block by block up to each horizon: the
  deviation is the time the relay spent in another state than planned.
  Unobserved relay time counts as possible deviation.
- Matched (at most 2 minutes, worst case): Forecast Error figures.
  Approximate (at most 5 minutes, at most 2 of them unobserved): separate
  approximate_stats. Counterfactual (more than 5 minutes seen): counted, not
  scored. Incomplete otherwise, by reason: relay history, assumed inputs at
  forecast time (an unknown window or electrical heat), no poll near the
  due time, or a missing reading.
- 2 minutes of unplanned boiler heat move a room by roughly half a 0.1 °C
  sensor step for the learned heat coefficients (0.05-0.12 °C per 5
  minutes); 5 minutes by about one step.
- Stored ledger statistics are versioned (2); 0.4.1 statistics used the
  looser rule and are dropped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

custom_components/heating_mpc/coordinator.py

@@ -895,9 +895,16 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]):
             _LOGGER.warning("MPC prediction failed: %s", err)
             status["status"] = "prediction_error"
             return status
+        # Inputs the forecast had to assume: such forecasts are not compared with reality.
+        uncertain = [f"window: {name}" for room in vals
+                     for name in (vent_inputs.get(room) or {}).get("unknown_windows", [])]
+        uncertain += [f"electrical heat: {self.zones[room].name}" for room in vals
+                      if internal[room] is None and room in self.extended
+                      and ZoneModel(self.learners[room], self.extended[room]).uses_extended]
         self._forecasts.record(now.timestamp(), vals, output["trajectory_15min"], output["plan_15min"],
                                self._revision(), {
             "strategy": "mpc_plan", "outside": outside, "wind": weather.get("wind"),
+            "uncertain_inputs": uncertain,
             "internal_half_life_minutes": float(self.config.get(CONF_INTERNAL_HALF_LIFE, 30)),
             "window_open_minutes": float(self.config[CONF_WINDOW_OPEN_MINUTES]),
             "internal_kw": {room: internal[room] for room in vals},

custom_components/heating_mpc/evaluation.py

@@ -371,11 +371,24 @@ def _stats(values: list[float], horizon: int, zones: int = 1) -> dict[str, Any]:
     }
 
 
-# A forecast's plan counts as executed when, in every 15-minute block, the share
-# of time the relay was on is within this of what the plan assumed (0 or 1).
-PLAN_TOLERANCE = 0.2
+# How far the relay may depart from a forecast's plan (seconds in another state than
+# planned, up to the horizon) for the forecast to be compared with what happened.
+# d seconds of boiler heat that the plan did not assume (or the reverse) move the
+# room by roughly heat x d / 300, where heat is the learned boiler coefficient
+# (typically 0.05-0.12 °C per 5 minutes with warm radiators). Up to 2 minutes keeps
+# that below half a typical 0.1 °C sensor step and absorbs timestamp jitter;
+# matched forecasts make the Forecast Error figures. Up to 5 minutes (one poll) is
+# about one sensor step: kept apart as approximate. Beyond, the relay did
+# something else: the forecast is counterfactual and is not scored. Relay time
+# that was not observed counts as possible deviation, and more than 2 minutes of it
+# leaves the forecast incomplete rather than approximate.
+MATCH_SECONDS = 120
+APPROXIMATE_SECONDS = 300
 DUE_TOLERANCE_SECONDS = 150  # half a poll: the poll closest to the forecast's due time
-VERDICTS = ("counterfactual", "unverifiable", "missing")
+VERDICTS = ("matched", "approximate", "counterfactual", "incomplete")
+# Why a forecast (or, for a missing reading, a zone's forecast) could not be compared.
+INCOMPLETE_REASONS = ("relay_history", "uncertain_inputs", "no_poll", "reading")
+LEDGER_VERSION = 2  # 1 (0.4.1) accepted plans within 20 % per 15-minute block
 
 
 class OperationalForecastLedger:
@@ -383,10 +396,10 @@ class OperationalForecastLedger:
 
     Nothing is replaced with hindsight: the start temperatures, trajectory, boiler
     plan and assumptions are those of the moment the forecast was made. Because the
-    plan is the MPC's proposal while Generic Thermostat controls the relay, a forecast
-    is only scored when the relay actually followed the plan up to that horizon;
-    otherwise it is counted as counterfactual. Pending forecasts live in memory;
-    the statistics are saved.
+    plan is the MPC's proposal while Generic Thermostat controls the relay, a
+    forecast is compared only when the relay actually followed that plan (see
+    MATCH_SECONDS), and not at all when it was made with an input it had to
+    assume. Pending forecasts live in memory; the statistics are saved.
     """
 
     def __init__(self, saved: Any = None) -> None:
@@ -394,8 +407,9 @@ class OperationalForecastLedger:
         # (interval end, seconds the relay was on or None if unknown, interval seconds)
         self._relay: deque[tuple[float, float | None, float]] = deque(maxlen=48)
         # UTC day -> horizon -> zone -> [count, sum error, sum |error|, sum error^2, sum |persistence error|]
-        self._days: dict[str, dict[str, dict[str, list[float]]]] = {}
-        # UTC day -> horizon -> verdict -> count of forecasts not scored, and why
+        self._days: dict[str, dict[str, dict[str, list[float]]]] = {}       # matched forecasts
+        self._approximate: dict[str, dict[str, dict[str, list[float]]]] = {}
+        # UTC day -> horizon -> verdict (incomplete ones as "incomplete:<reason>") -> count
         self._verdicts: dict[str, dict[str, dict[str, int]]] = {}
         self.load(saved)
 
@@ -420,20 +434,18 @@ class OperationalForecastLedger:
                 forecast["done"].add(horizon)
                 if index >= len(forecast["plan"]):
                     continue  # beyond this forecast's horizon
-                if now > due + DUE_TOLERANCE_SECONDS:
-                    self._count(day, horizon, "missing")  # no poll near the due time
-                    continue
-                verdict = self._verify(forecast, horizon)
-                if verdict:
+                verdict = self._classify(forecast, horizon, now)
                 self._count(day, horizon, verdict)
+                if verdict not in ("matched", "approximate"):
                     continue
+                store = self._days if verdict == "matched" else self._approximate
                 for zone, path in forecast["trajectory"].items():
                     measured, start = actual.get(zone), forecast["start"].get(zone)
                     if measured is None or start is None or len(path) <= index:
-                        self._count(day, horizon, "missing")
+                        self._count(day, horizon, "incomplete:reading")
                         continue
                     error = path[index] - measured
-                    bucket = self._days.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
+                    bucket = store.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
                         zone, [0, 0.0, 0.0, 0.0, 0.0])
                     bucket[0] += 1
                     bucket[1] += error
@@ -443,36 +455,60 @@ class OperationalForecastLedger:
             if len(forecast["done"]) == len(HORIZONS) or now > forecast["time"] + (HORIZONS[-1] + 5) * 60:
                 self._pending.remove(forecast)
         oldest = (datetime.fromtimestamp(now, timezone.utc).date() - timedelta(days=WINDOW_DAYS - 1)).isoformat()
-        for store in (self._days, self._verdicts):
+        for store in (self._days, self._approximate, self._verdicts):
             for old in [d for d in store if d < oldest]:
                 del store[old]
 
-    def _verify(self, forecast: dict[str, Any], horizon: int) -> str | None:
-        """None when the relay followed the plan up to `horizon`, else the reason."""
-        for block in range(horizon // 15):
-            start = forecast["time"] + block * 900
-            end = start + 900
-            on = covered = 0.0
+    def _classify(self, forecast: dict[str, Any], horizon: int, now: float) -> str:
+        if now > forecast["time"] + horizon * 60 + DUE_TOLERANCE_SECONDS:
+            return "incomplete:no_poll"
+        if forecast["assumptions"].get("uncertain_inputs"):
+            return "incomplete:uncertain_inputs"
+        deviation, unobserved = self.deviation(forecast, horizon)
+        if deviation > APPROXIMATE_SECONDS:
+            return "counterfactual"  # whatever happened in the unobserved part
+        # Otherwise judged by the worst case: unobserved time may all have deviated.
+        if deviation + unobserved <= MATCH_SECONDS:
+            return "matched"
+        if deviation + unobserved <= APPROXIMATE_SECONDS and unobserved <= MATCH_SECONDS:
+            return "approximate"
+        return "incomplete:relay_history"
+
+    def deviation(self, forecast: dict[str, Any], horizon: int) -> tuple[float, float]:
+        """Seconds up to `horizon` the relay was seen in another state than planned,
+        and seconds its state is not known.
+
+        Compared per 15-minute plan block, from the relay's on-time per poll interval,
+        itself integrated at the relay's own switch times. So timing counts: the same
+        total on-time at other times than planned is a deviation, while a switch a few
+        seconds off a block boundary is not.
+        """
+        start, end = forecast["time"], forecast["time"] + horizon * 60
+        deviation = covered = unobserved = 0.0
         for interval_end, on_seconds, seconds in self._relay:
-                overlap = min(end, interval_end) - max(start, interval_end - seconds)
-                if overlap <= 0:
+            lo, hi = max(start, interval_end - seconds), min(end, interval_end)
+            if hi <= lo:
                 continue
+            covered += hi - lo
             if on_seconds is None:
-                    return "unverifiable"
-                on += on_seconds * overlap / seconds
-                covered += overlap
-            if covered < 0.8 * 900:
-                return "unverifiable"
-            if abs(on / covered - forecast["plan"][block]) > PLAN_TOLERANCE:
-                return "counterfactual"
-        return None
+                unobserved += hi - lo
+                continue
+            share = on_seconds / seconds
+            t = lo
+            while t < hi:  # split at plan block boundaries
+                block = min(int((t - start) // 900), len(forecast["plan"]) - 1)
+                part_end = min(hi, start + (block + 1) * 900)
+                deviation += (part_end - t) * (share if not forecast["plan"][block] else 1.0 - share)
+                t = part_end
+        return deviation, unobserved + max(0.0, (end - start) - covered)
 
     def _count(self, day: str, horizon: int, verdict: str) -> None:
         counts = self._verdicts.setdefault(day, {}).setdefault(str(horizon), {})
         counts[verdict] = counts.get(verdict, 0) + 1
 
     def forget(self, zone: str) -> None:
-        for by_horizon in self._days.values():
+        for store in (self._days, self._approximate):
+            for by_horizon in store.values():
                 for by_zone in by_horizon.values():
                     by_zone.pop(zone, None)
         for forecast in self._pending:
@@ -485,58 +521,70 @@ class OperationalForecastLedger:
     def summary(self, zones: list[str]) -> dict[int, dict[str, Any]]:
         result = {}
         for horizon in HORIZONS:
-            per_zone: dict[str, list[float]] = {}
-            verdicts = {verdict: 0 for verdict in VERDICTS}
-            for by_horizon in self._days.values():
-                for zone, values in by_horizon.get(str(horizon), {}).items():
-                    if zone in zones:
-                        totals = per_zone.setdefault(zone, [0, 0.0, 0.0, 0.0, 0.0])
-                        for i, value in enumerate(values):
-                            totals[i] += value
+            per_zone = _sum_by_zone(self._days, horizon, zones)
+            approximate = _sum_by_zone(self._approximate, horizon, zones)
+            counts: dict[str, int] = {}
             for by_horizon in self._verdicts.values():
                 for verdict, count in by_horizon.get(str(horizon), {}).items():
-                    verdicts[verdict] = verdicts.get(verdict, 0) + count
+                    counts[verdict] = counts.get(verdict, 0) + count
+            reasons = {reason: counts.get(f"incomplete:{reason}", 0) for reason in INCOMPLETE_REASONS}
             overall = [sum(values[i] for values in per_zone.values()) for i in range(5)]
-            result[horizon] = {**_operational_stats(overall, horizon, len(per_zone)), **verdicts,
-                               "zones": {zone: _operational_stats(values, horizon)
-                                         for zone, values in per_zone.items()}}
+            approx_overall = [sum(values[i] for values in approximate.values()) for i in range(5)]
+            result[horizon] = {
+                **_operational_stats(overall, horizon, len(per_zone)),
+                # Forecasts by how the relay compared with their plan.
+                "matched": counts.get("matched", 0), "approximate": counts.get("approximate", 0),
+                "counterfactual": counts.get("counterfactual", 0), "incomplete": sum(reasons.values()),
+                "incomplete_reasons": reasons,
+                "approximate_stats": _operational_stats(approx_overall, horizon, len(approximate)),
+                "zones": {zone: _operational_stats(values, horizon) for zone, values in per_zone.items()},
+            }
         return result
 
     def as_dict(self) -> dict[str, Any]:
-        return {"days": {day: {h: {zone: [v[0], *(round(x, 6) for x in v[1:])] for zone, v in by_zone.items()}
-                               for h, by_zone in by_horizon.items()} for day, by_horizon in self._days.items()},
-                "verdicts": self._verdicts}
+        def rounded(store):
+            return {day: {h: {zone: [v[0], *(round(x, 6) for x in v[1:])] for zone, v in by_zone.items()}
+                          for h, by_zone in by_horizon.items()} for day, by_horizon in store.items()}
+        return {"version": LEDGER_VERSION, "days": rounded(self._days),
+                "approximate": rounded(self._approximate), "verdicts": self._verdicts}
 
     def load(self, data: Any) -> None:
-        self._days, self._verdicts = {}, {}
-        if not isinstance(data, dict):
+        self._days, self._approximate, self._verdicts = {}, {}, {}
+        # 0.4.1 statistics used a looser plan criterion: they are not comparable.
+        if not isinstance(data, dict) or data.get("version") != LEDGER_VERSION:
             return
-        horizons = {str(h) for h in HORIZONS}
-        days = data.get("days") if isinstance(data.get("days"), dict) else {}
-        for day, by_horizon in days.items():
-            if not _valid_day(day) or not isinstance(by_horizon, dict):
-                continue
-            for horizon, by_zone in by_horizon.items():
-                if horizon not in horizons or not isinstance(by_zone, dict):
-                    continue
-                for zone, values in by_zone.items():
-                    if (isinstance(zone, str) and isinstance(values, list) and len(values) == 5
-                            and type(values[0]) is int and values[0] >= 0
+        for name, store in (("days", self._days), ("approximate", self._approximate)):
+            for day, horizon, zone, values in _entries(data.get(name)):
+                if (isinstance(values, list) and len(values) == 5 and type(values[0]) is int and values[0] >= 0
                         and all(type(v) in (int, float) and isfinite(v) for v in values[1:])
                         and min(values[2:]) >= 0):
-                        self._days.setdefault(day, {}).setdefault(horizon, {})[zone] = [
+                    store.setdefault(day, {}).setdefault(horizon, {})[zone] = [
                         values[0], *(float(v) for v in values[1:])]
+        known = {"matched", "approximate", "counterfactual", *(f"incomplete:{r}" for r in INCOMPLETE_REASONS)}
         verdicts = data.get("verdicts") if isinstance(data.get("verdicts"), dict) else {}
+        horizons = {str(h) for h in HORIZONS}
         for day, by_horizon in verdicts.items():
             if not _valid_day(day) or not isinstance(by_horizon, dict):
                 continue
             for horizon, counts in by_horizon.items():
                 if horizon in horizons and isinstance(counts, dict):
-                    clean = {k: v for k, v in counts.items() if k in VERDICTS and type(v) is int and v >= 0}
+                    clean = {k: v for k, v in counts.items() if k in known and type(v) is int and v >= 0}
                     if clean:
                         self._verdicts.setdefault(day, {})[horizon] = clean
 
 
+def _sum_by_zone(store: dict[str, dict[str, dict[str, list[float]]]], horizon: int,
+                 zones: list[str]) -> dict[str, list[float]]:
+    totals: dict[str, list[float]] = {}
+    for by_horizon in store.values():
+        for zone, values in by_horizon.get(str(horizon), {}).items():
+            if zone in zones:
+                bucket = totals.setdefault(zone, [0, 0.0, 0.0, 0.0, 0.0])
+                for i, value in enumerate(values):
+                    bucket[i] += value
+    return totals
+
+
 def _valid_day(day: Any) -> bool:
     try:
         datetime.strptime(day, "%Y-%m-%d")

custom_components/heating_mpc/sensor.py

@@ -229,7 +229,8 @@ class MPCOperationalForecastError(CoordinatorEntity, SensorEntity):
             horizons[f"{horizon}_min"] = {**{k: v for k, v in stats.items() if k != "zones"},
                                           "zones": {names.get(z, z): v for z, v in stats.get("zones", {}).items()}}
         return {
-            "kind": "forecast accuracy: published forecasts, scored only where the relay followed the plan",
+            "kind": ("forecast accuracy: published forecasts, scored only where the relay matched the plan "
+                     "(at most 2 minutes in another state); approximate matches (up to 5 minutes) apart"),
             "horizons": horizons,
             "window_days": WINDOW_DAYS,
             "unit_of_attributes": "°C",

tests/test_coordinator.py

@@ -1329,8 +1329,24 @@ class BoundaryTests(unittest.IsolatedAsyncioTestCase):
                 # Longer horizons are not due yet.
                 self.assertEqual(data['operational_forecast'][30]['samples'], 0)
                 stored = self.c._payload('sensor.outside')['operational_evaluation']
-                self.assertEqual(sum(sum(v.get('15', {}).values()) for v in stored['verdicts'].values()),
-                                 0 if follows else 1)
+                counts = {}
+                for by_horizon in stored['verdicts'].values():
+                    for name, count in by_horizon.get('15', {}).items():
+                        counts[name] = counts.get(name, 0) + count
+                self.assertEqual(counts, {'matched': 1} if follows else {'counterfactual': 1})
+
+    async def test_a_forecast_made_with_an_unknown_window_is_never_scored(self):
+        self.ready_office()
+        self.set_state('sensor.room', '23.5', unit_of_measurement='°C')  # plan: off, as the relay
+        self.states['binary_sensor.office_window'].state = 'unavailable'
+        first = await self.sample()
+        self.assertEqual(first['status'], 'ready')
+        self.assertEqual(self.c._forecasts._pending[-1]['assumptions']['uncertain_inputs'], ['window: Window'])
+        for _ in range(3):
+            self.fresh('sensor.room', 'sensor.outside', 'sensor.ups_power')
+            data = await self.sample()
+        fifteen = data['operational_forecast'][15]
+        self.assertEqual((fifteen['samples'], fifteen['incomplete_reasons']['uncertain_inputs']), (0, 1))
 
     async def test_unloading_during_save_or_optimization_publishes_nothing_new(self):
         self.ready_office()

tests/test_evaluation.py

@@ -159,7 +159,7 @@ class OperationalForecastTests(unittest.TestCase):
         for horizon, actual in ((15, 19.9), (30, 19.8), (60, 19.6), (120, 19.2)):
             with self.subTest(horizon=horizon):
                 s = stats[horizon]
-                self.assertEqual((s['samples'], s['counterfactual'], s['unverifiable']), (1, 0, 0))
+                self.assertEqual((s['samples'], s['matched'], s['counterfactual'], s['incomplete']), (1, 1, 0, 0))
                 self.assertAlmostEqual(s['mae'], 0, places=3)
                 self.assertAlmostEqual(s['persistence_mae'], 20.0 - actual, places=3)
                 self.assertEqual(s['improvement_vs_persistence'], 1.0)
@@ -179,12 +179,14 @@ class OperationalForecastTests(unittest.TestCase):
 
     def test_unknown_relay_history_or_missing_observations(self):
         stats = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: 20.0, self.TRAJECTORY, unknown_relay_at=2)
-        self.assertEqual((stats[15]['samples'], stats[15]['unverifiable'], stats[60]['unverifiable']), (0, 1, 1))
+        self.assertEqual((stats[15]['samples'], stats[15]['incomplete_reasons']['relay_history'],
+                          stats[60]['incomplete']), (0, 1, 1))
         missing = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: None if k == 12 else 20.0, self.TRAJECTORY)
-        self.assertEqual((missing[60]['samples'], missing[60]['missing'], missing[30]['samples']), (0, 1, 1))
+        self.assertEqual((missing[60]['samples'], missing[60]['incomplete_reasons']['reading'],
+                          missing[30]['samples']), (0, 1, 1))
         # No poll near the due time (polls 11 to 13 missing): the 60-minute point cannot be scored.
         gap = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: 20.0, self.TRAJECTORY, skip=(11, 12, 13))
-        self.assertEqual((gap[60]['samples'], gap[60]['missing']), (0, 1))
+        self.assertEqual((gap[60]['samples'], gap[60]['incomplete_reasons']['no_poll']), (0, 1))
 
     def test_due_time_alignment(self):
         ledger = ev.OperationalForecastLedger()
@@ -207,8 +209,13 @@ class OperationalForecastTests(unittest.TestCase):
         self.assertEqual(restored.summary(['r'])[15]['counterfactual'], 1)
         restored.forget('r')
         self.assertEqual(restored.summary(['r'])[15]['samples'], 0)
-        for garbage in (None, [], {'days': {'x': {}}, 'verdicts': {'2026-10-27': {'15': {'bogus': 3}}}}):
-            self.assertEqual(ev.OperationalForecastLedger(garbage).as_dict(), {'days': {}, 'verdicts': {}})
+        empty = {'version': 2, 'days': {}, 'approximate': {}, 'verdicts': {}}
+        for garbage in (None, [], {'version': 2, 'days': {'x': {}}, 'verdicts': {'2026-10-27': {'15': {'bogus': 3}}}}):
+            self.assertEqual(ev.OperationalForecastLedger(garbage).as_dict(), empty)
+        # 0.4.1 statistics accepted plans within 20 % per block: not comparable, dropped.
+        old = {**stats_ledger.as_dict(), 'version': None}
+        old.pop('version')
+        self.assertEqual(ev.OperationalForecastLedger(old).as_dict(), empty)
 
 
 if __name__ == '__main__':
@@ -299,3 +306,108 @@ class IncompleteInputTests(unittest.TestCase):
         stats = ventilated([0.0] * 5, missing_end_at=(4,), zones=('a', 'b'))
         self.assertEqual((stats['rooms']['a']['samples'], stats['rooms']['a']['skipped']), (1, 1))
         self.assertEqual(stats['rooms']['b']['samples'], 2)
+
+
+def ledger_with(plan, on_seconds, temps=None, uncertain=None, polls=None):
+    """One forecast at T0; `on_seconds[k-1]` is the relay's on-time in the interval ending
+    at poll k (None: unknown). Returns the ledger."""
+    ledger = ev.OperationalForecastLedger()
+    trajectory = [20.0 + 0.1 * (b + 1) for b in range(len(plan))]
+    ledger.observe(T0, {'r': 20.0}, None, None)
+    ledger.record(T0, {'r': 20.0}, {'r': trajectory}, plan, 'rev1',
+                  {'strategy': 'mpc_plan', 'uncertain_inputs': uncertain or []})
+    for k in range(1, (polls or len(on_seconds)) + 1):
+        on = on_seconds[k - 1] if k - 1 < len(on_seconds) else 0.0
+        ledger.observe(T0 + 300 * k, {'r': (temps or (lambda k: 20.0 + k / 30))(k)}, on, 300.0)
+    return ledger
+
+
+def verdict(ledger, horizon):
+    stats = ledger.summary(['r'])[horizon]
+    for name in ('matched', 'approximate', 'counterfactual'):
+        if stats[name]:
+            return name
+    return next(f'incomplete:{r}' for r, n in stats['incomplete_reasons'].items() if n)
+
+
+class PlanMatchTests(unittest.TestCase):
+    """Forecasts are compared with reality only when the relay did what their plan assumed."""
+
+    ON, OFF = 300.0, 0.0
+
+    def test_exact_match_is_scored(self):
+        ledger = ledger_with([1, 1, 0, 0], [self.ON] * 6 + [self.OFF] * 6)
+        self.assertEqual([verdict(ledger, h) for h in (15, 30, 60)], ['matched'] * 3)
+        self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 1, 0, 0]}, 60), (0.0, 0.0))
+        stats = ledger.summary(['r'])[60]
+        self.assertEqual((stats['samples'], stats['approximate_stats']['samples']), (1, 0))
+
+    def test_small_transition_jitter_still_matches(self):
+        # Planned off at 30 minutes; the relay switched off 40 s later.
+        ledger = ledger_with([1, 1, 0, 0], [self.ON] * 6 + [40.0] + [self.OFF] * 5)
+        self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 1, 0, 0]}, 60), (40.0, 0.0))
+        self.assertEqual(verdict(ledger, 60), 'matched')
+
+    def test_twelve_of_fifteen_planned_minutes_is_only_approximate(self):
+        ledger = ledger_with([1, 0], [self.ON, self.ON, 120.0, self.OFF, self.OFF, self.OFF])
+        self.assertEqual(verdict(ledger, 15), 'approximate')
+        stats = ledger.summary(['r'])[15]
+        # Not in the main figures; kept apart.
+        self.assertEqual((stats['samples'], stats['mae'], stats['approximate']), (0, None, 1))
+        self.assertEqual(stats['approximate_stats']['samples'], 1)
+
+    def test_same_on_time_at_another_time_is_counterfactual(self):
+        # 15 minutes on as planned, but in the second block instead of the first.
+        ledger = ledger_with([1, 0, 0, 0], [self.OFF] * 3 + [self.ON] * 3 + [self.OFF] * 6)
+        self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 0, 0, 0]}, 30), (1800.0, 0.0))
+        self.assertEqual([verdict(ledger, h) for h in (15, 30, 60)], ['counterfactual'] * 3)
+
+    def test_early_shutdown_and_unexpected_activation(self):
+        early = ledger_with([1, 1, 1, 1], [self.ON] * 4 + [self.OFF] * 8)   # off after 20 of 60 minutes
+        self.assertEqual([verdict(early, h) for h in (15, 30, 60)], ['matched', 'counterfactual', 'counterfactual'])
+        brief = ledger_with([0, 0, 0, 0], [self.OFF] * 4 + [240.0] + [self.OFF] * 7)  # 4 minutes on
+        self.assertEqual([verdict(brief, h) for h in (15, 30)], ['matched', 'approximate'])
+        long = ledger_with([0, 0, 0, 0], [self.OFF] * 4 + [self.ON, self.ON] + [self.OFF] * 6)
+        self.assertEqual(verdict(long, 30), 'counterfactual')
+
+    def test_several_cycles_followed_closely_match(self):
+        plan = [1, 0, 1, 0, 1, 0, 1, 0]
+        relay = []
+        for block in plan:
+            relay += [self.ON] * 3 if block else [30.0, self.OFF, self.OFF]  # each off 30 s late
+        ledger = ledger_with(plan, relay)
+        self.assertEqual(ledger.deviation({'time': T0, 'plan': plan}, 120)[0], 120.0)  # four transitions
+        self.assertEqual(verdict(ledger, 60), 'matched')
+        self.assertEqual(verdict(ledger, 120), 'matched')
+
+    def test_missing_relay_history_is_incomplete_not_matched(self):
+        ledger = ledger_with([0, 0, 0, 0], [self.OFF, None] + [self.OFF] * 10)
+        self.assertEqual([verdict(ledger, h) for h in (15, 60)], ['incomplete:relay_history'] * 2)
+        # A plan already contradicted is counterfactual whatever the gap hides.
+        ledger = ledger_with([1, 1, 1, 1], [None] + [self.OFF] * 11)
+        self.assertEqual(verdict(ledger, 30), 'counterfactual')
+
+    def test_thermostat_disagreeing_is_counterfactual_never_an_error(self):
+        ledger = ledger_with([1] * 4, [self.OFF] * 12, temps=lambda k: 20.0)
+        stats = ledger.summary(['r'])
+        self.assertEqual([stats[h]['counterfactual'] for h in (15, 30, 60)], [1, 1, 1])
+        self.assertEqual([stats[h]['samples'] for h in (15, 30, 60)], [0, 0, 0])
+
+    def test_forecasts_made_with_assumed_inputs_are_incomplete(self):
+        ledger = ledger_with([0] * 4, [self.OFF] * 12, uncertain=['window: Bedroom'])
+        self.assertEqual(verdict(ledger, 15), 'incomplete:uncertain_inputs')
+        self.assertEqual(ledger.summary(['r'])[15]['samples'], 0)
+
+    def test_counts_over_several_forecasts(self):
+        ledger = ev.OperationalForecastLedger()
+        ledger.observe(T0, {'r': 20.0}, None, None)
+        for k, plan in enumerate(([0, 0], [1, 1], [0, 0])):   # one forecast per poll, relay always off
+            ledger.record(T0 + 300 * k, {'r': 20.0}, {'r': [20.0, 20.0]}, plan, 'rev1', {})
+            ledger.observe(T0 + 300 * (k + 1), {'r': 20.0}, 0.0, 300.0)
+        for k in range(4, 12):
+            ledger.observe(T0 + 300 * k, {'r': 20.0}, 0.0, 300.0)
+        stats = ledger.summary(['r'])[15]
+        self.assertEqual((stats['matched'], stats['counterfactual'], stats['samples'], stats['incomplete']),
+                         (2, 1, 2, 0))
+        restored = ev.OperationalForecastLedger(ledger.as_dict())
+        self.assertEqual(restored.summary(['r']), ledger.summary(['r']))