From 0b82a8e26cba4aad949a3261a9f84c575768cc89 Mon Sep 17 00:00:00 2001 From: Andrei Buzera Date: Tue, 27 Oct 2026 04:50:58 +0200 Subject: [PATCH] Score a forecast only when the relay matched its plan in time, not just in share MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reproduced: a forecast planning 15 minutes of heat was scored although the relay was on for 12 (the 0.4.1 rule allowed 20 % per 15-minute block), and the same on-time placed elsewhere in a block passed as well. - The relay's on-time per poll interval, integrated at its own switch times, is compared with the plan block by block up to each horizon: the deviation is the time the relay spent in another state than planned. Unobserved relay time counts as possible deviation. - Matched (at most 2 minutes, worst case): Forecast Error figures. Approximate (at most 5 minutes, at most 2 of them unobserved): separate approximate_stats. Counterfactual (more than 5 minutes seen): counted, not scored. Incomplete otherwise, by reason: relay history, assumed inputs at forecast time (an unknown window or electrical heat), no poll near the due time, or a missing reading. - 2 minutes of unplanned boiler heat move a room by roughly half a 0.1 °C sensor step for the learned heat coefficients (0.05-0.12 °C per 5 minutes); 5 minutes by about one step. - Stored ledger statistics are versioned (2); 0.4.1 statistics used the looser rule and are dropped. Co-Authored-By: Claude Opus 5.5 --- custom_components/heating_mpc/coordinator.py | 7 + custom_components/heating_mpc/evaluation.py | 196 ++++++++++++------- custom_components/heating_mpc/sensor.py | 3 +- tests/test_coordinator.py | 20 +- tests/test_evaluation.py | 124 +++++++++++- 5 files changed, 267 insertions(+), 83 deletions(-) diff --git a/custom_components/heating_mpc/coordinator.py b/custom_components/heating_mpc/coordinator.py index e4543a7..a9b152c 100644 --- a/custom_components/heating_mpc/coordinator.py +++ b/custom_components/heating_mpc/coordinator.py @@ -895,9 +895,16 @@ class HeatingMPCCoordinator(DataUpdateCoordinator[dict[str, Any]]): _LOGGER.warning("MPC prediction failed: %s", err) status["status"] = "prediction_error" return status + # Inputs the forecast had to assume: such forecasts are not compared with reality. + uncertain = [f"window: {name}" for room in vals + for name in (vent_inputs.get(room) or {}).get("unknown_windows", [])] + uncertain += [f"electrical heat: {self.zones[room].name}" for room in vals + if internal[room] is None and room in self.extended + and ZoneModel(self.learners[room], self.extended[room]).uses_extended] self._forecasts.record(now.timestamp(), vals, output["trajectory_15min"], output["plan_15min"], self._revision(), { "strategy": "mpc_plan", "outside": outside, "wind": weather.get("wind"), + "uncertain_inputs": uncertain, "internal_half_life_minutes": float(self.config.get(CONF_INTERNAL_HALF_LIFE, 30)), "window_open_minutes": float(self.config[CONF_WINDOW_OPEN_MINUTES]), "internal_kw": {room: internal[room] for room in vals}, diff --git a/custom_components/heating_mpc/evaluation.py b/custom_components/heating_mpc/evaluation.py index f8138f3..61a2236 100644 --- a/custom_components/heating_mpc/evaluation.py +++ b/custom_components/heating_mpc/evaluation.py @@ -371,11 +371,24 @@ def _stats(values: list[float], horizon: int, zones: int = 1) -> dict[str, Any]: } -# A forecast's plan counts as executed when, in every 15-minute block, the share -# of time the relay was on is within this of what the plan assumed (0 or 1). -PLAN_TOLERANCE = 0.2 +# How far the relay may depart from a forecast's plan (seconds in another state than +# planned, up to the horizon) for the forecast to be compared with what happened. +# d seconds of boiler heat that the plan did not assume (or the reverse) move the +# room by roughly heat x d / 300, where heat is the learned boiler coefficient +# (typically 0.05-0.12 °C per 5 minutes with warm radiators). Up to 2 minutes keeps +# that below half a typical 0.1 °C sensor step and absorbs timestamp jitter; +# matched forecasts make the Forecast Error figures. Up to 5 minutes (one poll) is +# about one sensor step: kept apart as approximate. Beyond, the relay did +# something else: the forecast is counterfactual and is not scored. Relay time +# that was not observed counts as possible deviation, and more than 2 minutes of it +# leaves the forecast incomplete rather than approximate. +MATCH_SECONDS = 120 +APPROXIMATE_SECONDS = 300 DUE_TOLERANCE_SECONDS = 150 # half a poll: the poll closest to the forecast's due time -VERDICTS = ("counterfactual", "unverifiable", "missing") +VERDICTS = ("matched", "approximate", "counterfactual", "incomplete") +# Why a forecast (or, for a missing reading, a zone's forecast) could not be compared. +INCOMPLETE_REASONS = ("relay_history", "uncertain_inputs", "no_poll", "reading") +LEDGER_VERSION = 2 # 1 (0.4.1) accepted plans within 20 % per 15-minute block class OperationalForecastLedger: @@ -383,10 +396,10 @@ class OperationalForecastLedger: Nothing is replaced with hindsight: the start temperatures, trajectory, boiler plan and assumptions are those of the moment the forecast was made. Because the - plan is the MPC's proposal while Generic Thermostat controls the relay, a forecast - is only scored when the relay actually followed the plan up to that horizon; - otherwise it is counted as counterfactual. Pending forecasts live in memory; - the statistics are saved. + plan is the MPC's proposal while Generic Thermostat controls the relay, a + forecast is compared only when the relay actually followed that plan (see + MATCH_SECONDS), and not at all when it was made with an input it had to + assume. Pending forecasts live in memory; the statistics are saved. """ def __init__(self, saved: Any = None) -> None: @@ -394,8 +407,9 @@ class OperationalForecastLedger: # (interval end, seconds the relay was on or None if unknown, interval seconds) self._relay: deque[tuple[float, float | None, float]] = deque(maxlen=48) # UTC day -> horizon -> zone -> [count, sum error, sum |error|, sum error^2, sum |persistence error|] - self._days: dict[str, dict[str, dict[str, list[float]]]] = {} - # UTC day -> horizon -> verdict -> count of forecasts not scored, and why + self._days: dict[str, dict[str, dict[str, list[float]]]] = {} # matched forecasts + self._approximate: dict[str, dict[str, dict[str, list[float]]]] = {} + # UTC day -> horizon -> verdict (incomplete ones as "incomplete:") -> count self._verdicts: dict[str, dict[str, dict[str, int]]] = {} self.load(saved) @@ -420,20 +434,18 @@ class OperationalForecastLedger: forecast["done"].add(horizon) if index >= len(forecast["plan"]): continue # beyond this forecast's horizon - if now > due + DUE_TOLERANCE_SECONDS: - self._count(day, horizon, "missing") # no poll near the due time - continue - verdict = self._verify(forecast, horizon) - if verdict: - self._count(day, horizon, verdict) + verdict = self._classify(forecast, horizon, now) + self._count(day, horizon, verdict) + if verdict not in ("matched", "approximate"): continue + store = self._days if verdict == "matched" else self._approximate for zone, path in forecast["trajectory"].items(): measured, start = actual.get(zone), forecast["start"].get(zone) if measured is None or start is None or len(path) <= index: - self._count(day, horizon, "missing") + self._count(day, horizon, "incomplete:reading") continue error = path[index] - measured - bucket = self._days.setdefault(day, {}).setdefault(str(horizon), {}).setdefault( + bucket = store.setdefault(day, {}).setdefault(str(horizon), {}).setdefault( zone, [0, 0.0, 0.0, 0.0, 0.0]) bucket[0] += 1 bucket[1] += error @@ -443,38 +455,62 @@ class OperationalForecastLedger: if len(forecast["done"]) == len(HORIZONS) or now > forecast["time"] + (HORIZONS[-1] + 5) * 60: self._pending.remove(forecast) oldest = (datetime.fromtimestamp(now, timezone.utc).date() - timedelta(days=WINDOW_DAYS - 1)).isoformat() - for store in (self._days, self._verdicts): + for store in (self._days, self._approximate, self._verdicts): for old in [d for d in store if d < oldest]: del store[old] - def _verify(self, forecast: dict[str, Any], horizon: int) -> str | None: - """None when the relay followed the plan up to `horizon`, else the reason.""" - for block in range(horizon // 15): - start = forecast["time"] + block * 900 - end = start + 900 - on = covered = 0.0 - for interval_end, on_seconds, seconds in self._relay: - overlap = min(end, interval_end) - max(start, interval_end - seconds) - if overlap <= 0: - continue - if on_seconds is None: - return "unverifiable" - on += on_seconds * overlap / seconds - covered += overlap - if covered < 0.8 * 900: - return "unverifiable" - if abs(on / covered - forecast["plan"][block]) > PLAN_TOLERANCE: - return "counterfactual" - return None + def _classify(self, forecast: dict[str, Any], horizon: int, now: float) -> str: + if now > forecast["time"] + horizon * 60 + DUE_TOLERANCE_SECONDS: + return "incomplete:no_poll" + if forecast["assumptions"].get("uncertain_inputs"): + return "incomplete:uncertain_inputs" + deviation, unobserved = self.deviation(forecast, horizon) + if deviation > APPROXIMATE_SECONDS: + return "counterfactual" # whatever happened in the unobserved part + # Otherwise judged by the worst case: unobserved time may all have deviated. + if deviation + unobserved <= MATCH_SECONDS: + return "matched" + if deviation + unobserved <= APPROXIMATE_SECONDS and unobserved <= MATCH_SECONDS: + return "approximate" + return "incomplete:relay_history" + + def deviation(self, forecast: dict[str, Any], horizon: int) -> tuple[float, float]: + """Seconds up to `horizon` the relay was seen in another state than planned, + and seconds its state is not known. + + Compared per 15-minute plan block, from the relay's on-time per poll interval, + itself integrated at the relay's own switch times. So timing counts: the same + total on-time at other times than planned is a deviation, while a switch a few + seconds off a block boundary is not. + """ + start, end = forecast["time"], forecast["time"] + horizon * 60 + deviation = covered = unobserved = 0.0 + for interval_end, on_seconds, seconds in self._relay: + lo, hi = max(start, interval_end - seconds), min(end, interval_end) + if hi <= lo: + continue + covered += hi - lo + if on_seconds is None: + unobserved += hi - lo + continue + share = on_seconds / seconds + t = lo + while t < hi: # split at plan block boundaries + block = min(int((t - start) // 900), len(forecast["plan"]) - 1) + part_end = min(hi, start + (block + 1) * 900) + deviation += (part_end - t) * (share if not forecast["plan"][block] else 1.0 - share) + t = part_end + return deviation, unobserved + max(0.0, (end - start) - covered) def _count(self, day: str, horizon: int, verdict: str) -> None: counts = self._verdicts.setdefault(day, {}).setdefault(str(horizon), {}) counts[verdict] = counts.get(verdict, 0) + 1 def forget(self, zone: str) -> None: - for by_horizon in self._days.values(): - for by_zone in by_horizon.values(): - by_zone.pop(zone, None) + for store in (self._days, self._approximate): + for by_horizon in store.values(): + for by_zone in by_horizon.values(): + by_zone.pop(zone, None) for forecast in self._pending: forecast["trajectory"].pop(zone, None) @@ -485,58 +521,70 @@ class OperationalForecastLedger: def summary(self, zones: list[str]) -> dict[int, dict[str, Any]]: result = {} for horizon in HORIZONS: - per_zone: dict[str, list[float]] = {} - verdicts = {verdict: 0 for verdict in VERDICTS} - for by_horizon in self._days.values(): - for zone, values in by_horizon.get(str(horizon), {}).items(): - if zone in zones: - totals = per_zone.setdefault(zone, [0, 0.0, 0.0, 0.0, 0.0]) - for i, value in enumerate(values): - totals[i] += value + per_zone = _sum_by_zone(self._days, horizon, zones) + approximate = _sum_by_zone(self._approximate, horizon, zones) + counts: dict[str, int] = {} for by_horizon in self._verdicts.values(): for verdict, count in by_horizon.get(str(horizon), {}).items(): - verdicts[verdict] = verdicts.get(verdict, 0) + count + counts[verdict] = counts.get(verdict, 0) + count + reasons = {reason: counts.get(f"incomplete:{reason}", 0) for reason in INCOMPLETE_REASONS} overall = [sum(values[i] for values in per_zone.values()) for i in range(5)] - result[horizon] = {**_operational_stats(overall, horizon, len(per_zone)), **verdicts, - "zones": {zone: _operational_stats(values, horizon) - for zone, values in per_zone.items()}} + approx_overall = [sum(values[i] for values in approximate.values()) for i in range(5)] + result[horizon] = { + **_operational_stats(overall, horizon, len(per_zone)), + # Forecasts by how the relay compared with their plan. + "matched": counts.get("matched", 0), "approximate": counts.get("approximate", 0), + "counterfactual": counts.get("counterfactual", 0), "incomplete": sum(reasons.values()), + "incomplete_reasons": reasons, + "approximate_stats": _operational_stats(approx_overall, horizon, len(approximate)), + "zones": {zone: _operational_stats(values, horizon) for zone, values in per_zone.items()}, + } return result def as_dict(self) -> dict[str, Any]: - return {"days": {day: {h: {zone: [v[0], *(round(x, 6) for x in v[1:])] for zone, v in by_zone.items()} - for h, by_zone in by_horizon.items()} for day, by_horizon in self._days.items()}, - "verdicts": self._verdicts} + def rounded(store): + return {day: {h: {zone: [v[0], *(round(x, 6) for x in v[1:])] for zone, v in by_zone.items()} + for h, by_zone in by_horizon.items()} for day, by_horizon in store.items()} + return {"version": LEDGER_VERSION, "days": rounded(self._days), + "approximate": rounded(self._approximate), "verdicts": self._verdicts} def load(self, data: Any) -> None: - self._days, self._verdicts = {}, {} - if not isinstance(data, dict): + self._days, self._approximate, self._verdicts = {}, {}, {} + # 0.4.1 statistics used a looser plan criterion: they are not comparable. + if not isinstance(data, dict) or data.get("version") != LEDGER_VERSION: return - horizons = {str(h) for h in HORIZONS} - days = data.get("days") if isinstance(data.get("days"), dict) else {} - for day, by_horizon in days.items(): - if not _valid_day(day) or not isinstance(by_horizon, dict): - continue - for horizon, by_zone in by_horizon.items(): - if horizon not in horizons or not isinstance(by_zone, dict): - continue - for zone, values in by_zone.items(): - if (isinstance(zone, str) and isinstance(values, list) and len(values) == 5 - and type(values[0]) is int and values[0] >= 0 - and all(type(v) in (int, float) and isfinite(v) for v in values[1:]) - and min(values[2:]) >= 0): - self._days.setdefault(day, {}).setdefault(horizon, {})[zone] = [ - values[0], *(float(v) for v in values[1:])] + for name, store in (("days", self._days), ("approximate", self._approximate)): + for day, horizon, zone, values in _entries(data.get(name)): + if (isinstance(values, list) and len(values) == 5 and type(values[0]) is int and values[0] >= 0 + and all(type(v) in (int, float) and isfinite(v) for v in values[1:]) + and min(values[2:]) >= 0): + store.setdefault(day, {}).setdefault(horizon, {})[zone] = [ + values[0], *(float(v) for v in values[1:])] + known = {"matched", "approximate", "counterfactual", *(f"incomplete:{r}" for r in INCOMPLETE_REASONS)} verdicts = data.get("verdicts") if isinstance(data.get("verdicts"), dict) else {} + horizons = {str(h) for h in HORIZONS} for day, by_horizon in verdicts.items(): if not _valid_day(day) or not isinstance(by_horizon, dict): continue for horizon, counts in by_horizon.items(): if horizon in horizons and isinstance(counts, dict): - clean = {k: v for k, v in counts.items() if k in VERDICTS and type(v) is int and v >= 0} + clean = {k: v for k, v in counts.items() if k in known and type(v) is int and v >= 0} if clean: self._verdicts.setdefault(day, {})[horizon] = clean +def _sum_by_zone(store: dict[str, dict[str, dict[str, list[float]]]], horizon: int, + zones: list[str]) -> dict[str, list[float]]: + totals: dict[str, list[float]] = {} + for by_horizon in store.values(): + for zone, values in by_horizon.get(str(horizon), {}).items(): + if zone in zones: + bucket = totals.setdefault(zone, [0, 0.0, 0.0, 0.0, 0.0]) + for i, value in enumerate(values): + bucket[i] += value + return totals + + def _valid_day(day: Any) -> bool: try: datetime.strptime(day, "%Y-%m-%d") diff --git a/custom_components/heating_mpc/sensor.py b/custom_components/heating_mpc/sensor.py index 6021aa2..101fc45 100644 --- a/custom_components/heating_mpc/sensor.py +++ b/custom_components/heating_mpc/sensor.py @@ -229,7 +229,8 @@ class MPCOperationalForecastError(CoordinatorEntity, SensorEntity): horizons[f"{horizon}_min"] = {**{k: v for k, v in stats.items() if k != "zones"}, "zones": {names.get(z, z): v for z, v in stats.get("zones", {}).items()}} return { - "kind": "forecast accuracy: published forecasts, scored only where the relay followed the plan", + "kind": ("forecast accuracy: published forecasts, scored only where the relay matched the plan " + "(at most 2 minutes in another state); approximate matches (up to 5 minutes) apart"), "horizons": horizons, "window_days": WINDOW_DAYS, "unit_of_attributes": "°C", diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py index dc29678..26d7b04 100644 --- a/tests/test_coordinator.py +++ b/tests/test_coordinator.py @@ -1329,8 +1329,24 @@ class BoundaryTests(unittest.IsolatedAsyncioTestCase): # Longer horizons are not due yet. self.assertEqual(data['operational_forecast'][30]['samples'], 0) stored = self.c._payload('sensor.outside')['operational_evaluation'] - self.assertEqual(sum(sum(v.get('15', {}).values()) for v in stored['verdicts'].values()), - 0 if follows else 1) + counts = {} + for by_horizon in stored['verdicts'].values(): + for name, count in by_horizon.get('15', {}).items(): + counts[name] = counts.get(name, 0) + count + self.assertEqual(counts, {'matched': 1} if follows else {'counterfactual': 1}) + + async def test_a_forecast_made_with_an_unknown_window_is_never_scored(self): + self.ready_office() + self.set_state('sensor.room', '23.5', unit_of_measurement='°C') # plan: off, as the relay + self.states['binary_sensor.office_window'].state = 'unavailable' + first = await self.sample() + self.assertEqual(first['status'], 'ready') + self.assertEqual(self.c._forecasts._pending[-1]['assumptions']['uncertain_inputs'], ['window: Window']) + for _ in range(3): + self.fresh('sensor.room', 'sensor.outside', 'sensor.ups_power') + data = await self.sample() + fifteen = data['operational_forecast'][15] + self.assertEqual((fifteen['samples'], fifteen['incomplete_reasons']['uncertain_inputs']), (0, 1)) async def test_unloading_during_save_or_optimization_publishes_nothing_new(self): self.ready_office() diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 4302c7a..341aaec 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -159,7 +159,7 @@ class OperationalForecastTests(unittest.TestCase): for horizon, actual in ((15, 19.9), (30, 19.8), (60, 19.6), (120, 19.2)): with self.subTest(horizon=horizon): s = stats[horizon] - self.assertEqual((s['samples'], s['counterfactual'], s['unverifiable']), (1, 0, 0)) + self.assertEqual((s['samples'], s['matched'], s['counterfactual'], s['incomplete']), (1, 1, 0, 0)) self.assertAlmostEqual(s['mae'], 0, places=3) self.assertAlmostEqual(s['persistence_mae'], 20.0 - actual, places=3) self.assertEqual(s['improvement_vs_persistence'], 1.0) @@ -179,12 +179,14 @@ class OperationalForecastTests(unittest.TestCase): def test_unknown_relay_history_or_missing_observations(self): stats = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: 20.0, self.TRAJECTORY, unknown_relay_at=2) - self.assertEqual((stats[15]['samples'], stats[15]['unverifiable'], stats[60]['unverifiable']), (0, 1, 1)) + self.assertEqual((stats[15]['samples'], stats[15]['incomplete_reasons']['relay_history'], + stats[60]['incomplete']), (0, 1, 1)) missing = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: None if k == 12 else 20.0, self.TRAJECTORY) - self.assertEqual((missing[60]['samples'], missing[60]['missing'], missing[30]['samples']), (0, 1, 1)) + self.assertEqual((missing[60]['samples'], missing[60]['incomplete_reasons']['reading'], + missing[30]['samples']), (0, 1, 1)) # No poll near the due time (polls 11 to 13 missing): the 60-minute point cannot be scored. gap = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: 20.0, self.TRAJECTORY, skip=(11, 12, 13)) - self.assertEqual((gap[60]['samples'], gap[60]['missing']), (0, 1)) + self.assertEqual((gap[60]['samples'], gap[60]['incomplete_reasons']['no_poll']), (0, 1)) def test_due_time_alignment(self): ledger = ev.OperationalForecastLedger() @@ -207,8 +209,13 @@ class OperationalForecastTests(unittest.TestCase): self.assertEqual(restored.summary(['r'])[15]['counterfactual'], 1) restored.forget('r') self.assertEqual(restored.summary(['r'])[15]['samples'], 0) - for garbage in (None, [], {'days': {'x': {}}, 'verdicts': {'2026-10-27': {'15': {'bogus': 3}}}}): - self.assertEqual(ev.OperationalForecastLedger(garbage).as_dict(), {'days': {}, 'verdicts': {}}) + empty = {'version': 2, 'days': {}, 'approximate': {}, 'verdicts': {}} + for garbage in (None, [], {'version': 2, 'days': {'x': {}}, 'verdicts': {'2026-10-27': {'15': {'bogus': 3}}}}): + self.assertEqual(ev.OperationalForecastLedger(garbage).as_dict(), empty) + # 0.4.1 statistics accepted plans within 20 % per block: not comparable, dropped. + old = {**stats_ledger.as_dict(), 'version': None} + old.pop('version') + self.assertEqual(ev.OperationalForecastLedger(old).as_dict(), empty) if __name__ == '__main__': @@ -299,3 +306,108 @@ class IncompleteInputTests(unittest.TestCase): stats = ventilated([0.0] * 5, missing_end_at=(4,), zones=('a', 'b')) self.assertEqual((stats['rooms']['a']['samples'], stats['rooms']['a']['skipped']), (1, 1)) self.assertEqual(stats['rooms']['b']['samples'], 2) + + +def ledger_with(plan, on_seconds, temps=None, uncertain=None, polls=None): + """One forecast at T0; `on_seconds[k-1]` is the relay's on-time in the interval ending + at poll k (None: unknown). Returns the ledger.""" + ledger = ev.OperationalForecastLedger() + trajectory = [20.0 + 0.1 * (b + 1) for b in range(len(plan))] + ledger.observe(T0, {'r': 20.0}, None, None) + ledger.record(T0, {'r': 20.0}, {'r': trajectory}, plan, 'rev1', + {'strategy': 'mpc_plan', 'uncertain_inputs': uncertain or []}) + for k in range(1, (polls or len(on_seconds)) + 1): + on = on_seconds[k - 1] if k - 1 < len(on_seconds) else 0.0 + ledger.observe(T0 + 300 * k, {'r': (temps or (lambda k: 20.0 + k / 30))(k)}, on, 300.0) + return ledger + + +def verdict(ledger, horizon): + stats = ledger.summary(['r'])[horizon] + for name in ('matched', 'approximate', 'counterfactual'): + if stats[name]: + return name + return next(f'incomplete:{r}' for r, n in stats['incomplete_reasons'].items() if n) + + +class PlanMatchTests(unittest.TestCase): + """Forecasts are compared with reality only when the relay did what their plan assumed.""" + + ON, OFF = 300.0, 0.0 + + def test_exact_match_is_scored(self): + ledger = ledger_with([1, 1, 0, 0], [self.ON] * 6 + [self.OFF] * 6) + self.assertEqual([verdict(ledger, h) for h in (15, 30, 60)], ['matched'] * 3) + self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 1, 0, 0]}, 60), (0.0, 0.0)) + stats = ledger.summary(['r'])[60] + self.assertEqual((stats['samples'], stats['approximate_stats']['samples']), (1, 0)) + + def test_small_transition_jitter_still_matches(self): + # Planned off at 30 minutes; the relay switched off 40 s later. + ledger = ledger_with([1, 1, 0, 0], [self.ON] * 6 + [40.0] + [self.OFF] * 5) + self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 1, 0, 0]}, 60), (40.0, 0.0)) + self.assertEqual(verdict(ledger, 60), 'matched') + + def test_twelve_of_fifteen_planned_minutes_is_only_approximate(self): + ledger = ledger_with([1, 0], [self.ON, self.ON, 120.0, self.OFF, self.OFF, self.OFF]) + self.assertEqual(verdict(ledger, 15), 'approximate') + stats = ledger.summary(['r'])[15] + # Not in the main figures; kept apart. + self.assertEqual((stats['samples'], stats['mae'], stats['approximate']), (0, None, 1)) + self.assertEqual(stats['approximate_stats']['samples'], 1) + + def test_same_on_time_at_another_time_is_counterfactual(self): + # 15 minutes on as planned, but in the second block instead of the first. + ledger = ledger_with([1, 0, 0, 0], [self.OFF] * 3 + [self.ON] * 3 + [self.OFF] * 6) + self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 0, 0, 0]}, 30), (1800.0, 0.0)) + self.assertEqual([verdict(ledger, h) for h in (15, 30, 60)], ['counterfactual'] * 3) + + def test_early_shutdown_and_unexpected_activation(self): + early = ledger_with([1, 1, 1, 1], [self.ON] * 4 + [self.OFF] * 8) # off after 20 of 60 minutes + self.assertEqual([verdict(early, h) for h in (15, 30, 60)], ['matched', 'counterfactual', 'counterfactual']) + brief = ledger_with([0, 0, 0, 0], [self.OFF] * 4 + [240.0] + [self.OFF] * 7) # 4 minutes on + self.assertEqual([verdict(brief, h) for h in (15, 30)], ['matched', 'approximate']) + long = ledger_with([0, 0, 0, 0], [self.OFF] * 4 + [self.ON, self.ON] + [self.OFF] * 6) + self.assertEqual(verdict(long, 30), 'counterfactual') + + def test_several_cycles_followed_closely_match(self): + plan = [1, 0, 1, 0, 1, 0, 1, 0] + relay = [] + for block in plan: + relay += [self.ON] * 3 if block else [30.0, self.OFF, self.OFF] # each off 30 s late + ledger = ledger_with(plan, relay) + self.assertEqual(ledger.deviation({'time': T0, 'plan': plan}, 120)[0], 120.0) # four transitions + self.assertEqual(verdict(ledger, 60), 'matched') + self.assertEqual(verdict(ledger, 120), 'matched') + + def test_missing_relay_history_is_incomplete_not_matched(self): + ledger = ledger_with([0, 0, 0, 0], [self.OFF, None] + [self.OFF] * 10) + self.assertEqual([verdict(ledger, h) for h in (15, 60)], ['incomplete:relay_history'] * 2) + # A plan already contradicted is counterfactual whatever the gap hides. + ledger = ledger_with([1, 1, 1, 1], [None] + [self.OFF] * 11) + self.assertEqual(verdict(ledger, 30), 'counterfactual') + + def test_thermostat_disagreeing_is_counterfactual_never_an_error(self): + ledger = ledger_with([1] * 4, [self.OFF] * 12, temps=lambda k: 20.0) + stats = ledger.summary(['r']) + self.assertEqual([stats[h]['counterfactual'] for h in (15, 30, 60)], [1, 1, 1]) + self.assertEqual([stats[h]['samples'] for h in (15, 30, 60)], [0, 0, 0]) + + def test_forecasts_made_with_assumed_inputs_are_incomplete(self): + ledger = ledger_with([0] * 4, [self.OFF] * 12, uncertain=['window: Bedroom']) + self.assertEqual(verdict(ledger, 15), 'incomplete:uncertain_inputs') + self.assertEqual(ledger.summary(['r'])[15]['samples'], 0) + + def test_counts_over_several_forecasts(self): + ledger = ev.OperationalForecastLedger() + ledger.observe(T0, {'r': 20.0}, None, None) + for k, plan in enumerate(([0, 0], [1, 1], [0, 0])): # one forecast per poll, relay always off + ledger.record(T0 + 300 * k, {'r': 20.0}, {'r': [20.0, 20.0]}, plan, 'rev1', {}) + ledger.observe(T0 + 300 * (k + 1), {'r': 20.0}, 0.0, 300.0) + for k in range(4, 12): + ledger.observe(T0 + 300 * k, {'r': 20.0}, 0.0, 300.0) + stats = ledger.summary(['r'])[15] + self.assertEqual((stats['matched'], stats['counterfactual'], stats['samples'], stats['incomplete']), + (2, 1, 2, 0)) + restored = ev.OperationalForecastLedger(ledger.as_dict()) + self.assertEqual(restored.summary(['r']), ledger.summary(['r']))