Score a forecast only when the relay matched its plan in time, not just in share
Reproduced: a forecast planning 15 minutes of heat was scored although the relay was on for 12 (the 0.4.1 rule allowed 20 % per 15-minute block), and the same on-time placed elsewhere in a block passed as well. - The relay's on-time per poll interval, integrated at its own switch times, is compared with the plan block by block up to each horizon: the deviation is the time the relay spent in another state than planned. Unobserved relay time counts as possible deviation. - Matched (at most 2 minutes, worst case): Forecast Error figures. Approximate (at most 5 minutes, at most 2 of them unobserved): separate approximate_stats. Counterfactual (more than 5 minutes seen): counted, not scored. Incomplete otherwise, by reason: relay history, assumed inputs at forecast time (an unknown window or electrical heat), no poll near the due time, or a missing reading. - 2 minutes of unplanned boiler heat move a room by roughly half a 0.1 °C sensor step for the learned heat coefficients (0.05-0.12 °C per 5 minutes); 5 minutes by about one step. - Stored ledger statistics are versioned (2); 0.4.1 statistics used the looser rule and are dropped. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
buzeraandClaude Opus 5.5 committed
1 parent
2cfba024ff
commit
0b82a8e26c
5 files changed
+267
-83
No files matched your search
+118
-6
@@ -159,7 +159,7 @@ class OperationalForecastTests(unittest.TestCase):
|
||||
for horizon, actual in ((15, 19.9), (30, 19.8), (60, 19.6), (120, 19.2)):
|
||||
with self.subTest(horizon=horizon):
|
||||
s = stats[horizon]
|
||||
self.assertEqual((s['samples'], s['counterfactual'], s['unverifiable']), (1, 0, 0))
|
||||
self.assertEqual((s['samples'], s['matched'], s['counterfactual'], s['incomplete']), (1, 1, 0, 0))
|
||||
self.assertAlmostEqual(s['mae'], 0, places=3)
|
||||
self.assertAlmostEqual(s['persistence_mae'], 20.0 - actual, places=3)
|
||||
self.assertEqual(s['improvement_vs_persistence'], 1.0)
|
||||
@@ -179,12 +179,14 @@ class OperationalForecastTests(unittest.TestCase):
|
||||
|
||||
def test_unknown_relay_history_or_missing_observations(self):
|
||||
stats = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: 20.0, self.TRAJECTORY, unknown_relay_at=2)
|
||||
self.assertEqual((stats[15]['samples'], stats[15]['unverifiable'], stats[60]['unverifiable']), (0, 1, 1))
|
||||
self.assertEqual((stats[15]['samples'], stats[15]['incomplete_reasons']['relay_history'],
|
||||
stats[60]['incomplete']), (0, 1, 1))
|
||||
missing = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: None if k == 12 else 20.0, self.TRAJECTORY)
|
||||
self.assertEqual((missing[60]['samples'], missing[60]['missing'], missing[30]['samples']), (0, 1, 1))
|
||||
self.assertEqual((missing[60]['samples'], missing[60]['incomplete_reasons']['reading'],
|
||||
missing[30]['samples']), (0, 1, 1))
|
||||
# No poll near the due time (polls 11 to 13 missing): the 60-minute point cannot be scored.
|
||||
gap = run_ledger(self.PLAN_OFF, lambda k: False, lambda k: 20.0, self.TRAJECTORY, skip=(11, 12, 13))
|
||||
self.assertEqual((gap[60]['samples'], gap[60]['missing']), (0, 1))
|
||||
self.assertEqual((gap[60]['samples'], gap[60]['incomplete_reasons']['no_poll']), (0, 1))
|
||||
|
||||
def test_due_time_alignment(self):
|
||||
ledger = ev.OperationalForecastLedger()
|
||||
@@ -207,8 +209,13 @@ class OperationalForecastTests(unittest.TestCase):
|
||||
self.assertEqual(restored.summary(['r'])[15]['counterfactual'], 1)
|
||||
restored.forget('r')
|
||||
self.assertEqual(restored.summary(['r'])[15]['samples'], 0)
|
||||
for garbage in (None, [], {'days': {'x': {}}, 'verdicts': {'2026-10-27': {'15': {'bogus': 3}}}}):
|
||||
self.assertEqual(ev.OperationalForecastLedger(garbage).as_dict(), {'days': {}, 'verdicts': {}})
|
||||
empty = {'version': 2, 'days': {}, 'approximate': {}, 'verdicts': {}}
|
||||
for garbage in (None, [], {'version': 2, 'days': {'x': {}}, 'verdicts': {'2026-10-27': {'15': {'bogus': 3}}}}):
|
||||
self.assertEqual(ev.OperationalForecastLedger(garbage).as_dict(), empty)
|
||||
# 0.4.1 statistics accepted plans within 20 % per block: not comparable, dropped.
|
||||
old = {**stats_ledger.as_dict(), 'version': None}
|
||||
old.pop('version')
|
||||
self.assertEqual(ev.OperationalForecastLedger(old).as_dict(), empty)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
@@ -299,3 +306,108 @@ class IncompleteInputTests(unittest.TestCase):
|
||||
stats = ventilated([0.0] * 5, missing_end_at=(4,), zones=('a', 'b'))
|
||||
self.assertEqual((stats['rooms']['a']['samples'], stats['rooms']['a']['skipped']), (1, 1))
|
||||
self.assertEqual(stats['rooms']['b']['samples'], 2)
|
||||
|
||||
|
||||
def ledger_with(plan, on_seconds, temps=None, uncertain=None, polls=None):
|
||||
"""One forecast at T0; `on_seconds[k-1]` is the relay's on-time in the interval ending
|
||||
at poll k (None: unknown). Returns the ledger."""
|
||||
ledger = ev.OperationalForecastLedger()
|
||||
trajectory = [20.0 + 0.1 * (b + 1) for b in range(len(plan))]
|
||||
ledger.observe(T0, {'r': 20.0}, None, None)
|
||||
ledger.record(T0, {'r': 20.0}, {'r': trajectory}, plan, 'rev1',
|
||||
{'strategy': 'mpc_plan', 'uncertain_inputs': uncertain or []})
|
||||
for k in range(1, (polls or len(on_seconds)) + 1):
|
||||
on = on_seconds[k - 1] if k - 1 < len(on_seconds) else 0.0
|
||||
ledger.observe(T0 + 300 * k, {'r': (temps or (lambda k: 20.0 + k / 30))(k)}, on, 300.0)
|
||||
return ledger
|
||||
|
||||
|
||||
def verdict(ledger, horizon):
|
||||
stats = ledger.summary(['r'])[horizon]
|
||||
for name in ('matched', 'approximate', 'counterfactual'):
|
||||
if stats[name]:
|
||||
return name
|
||||
return next(f'incomplete:{r}' for r, n in stats['incomplete_reasons'].items() if n)
|
||||
|
||||
|
||||
class PlanMatchTests(unittest.TestCase):
|
||||
"""Forecasts are compared with reality only when the relay did what their plan assumed."""
|
||||
|
||||
ON, OFF = 300.0, 0.0
|
||||
|
||||
def test_exact_match_is_scored(self):
|
||||
ledger = ledger_with([1, 1, 0, 0], [self.ON] * 6 + [self.OFF] * 6)
|
||||
self.assertEqual([verdict(ledger, h) for h in (15, 30, 60)], ['matched'] * 3)
|
||||
self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 1, 0, 0]}, 60), (0.0, 0.0))
|
||||
stats = ledger.summary(['r'])[60]
|
||||
self.assertEqual((stats['samples'], stats['approximate_stats']['samples']), (1, 0))
|
||||
|
||||
def test_small_transition_jitter_still_matches(self):
|
||||
# Planned off at 30 minutes; the relay switched off 40 s later.
|
||||
ledger = ledger_with([1, 1, 0, 0], [self.ON] * 6 + [40.0] + [self.OFF] * 5)
|
||||
self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 1, 0, 0]}, 60), (40.0, 0.0))
|
||||
self.assertEqual(verdict(ledger, 60), 'matched')
|
||||
|
||||
def test_twelve_of_fifteen_planned_minutes_is_only_approximate(self):
|
||||
ledger = ledger_with([1, 0], [self.ON, self.ON, 120.0, self.OFF, self.OFF, self.OFF])
|
||||
self.assertEqual(verdict(ledger, 15), 'approximate')
|
||||
stats = ledger.summary(['r'])[15]
|
||||
# Not in the main figures; kept apart.
|
||||
self.assertEqual((stats['samples'], stats['mae'], stats['approximate']), (0, None, 1))
|
||||
self.assertEqual(stats['approximate_stats']['samples'], 1)
|
||||
|
||||
def test_same_on_time_at_another_time_is_counterfactual(self):
|
||||
# 15 minutes on as planned, but in the second block instead of the first.
|
||||
ledger = ledger_with([1, 0, 0, 0], [self.OFF] * 3 + [self.ON] * 3 + [self.OFF] * 6)
|
||||
self.assertEqual(ledger.deviation({'time': T0, 'plan': [1, 0, 0, 0]}, 30), (1800.0, 0.0))
|
||||
self.assertEqual([verdict(ledger, h) for h in (15, 30, 60)], ['counterfactual'] * 3)
|
||||
|
||||
def test_early_shutdown_and_unexpected_activation(self):
|
||||
early = ledger_with([1, 1, 1, 1], [self.ON] * 4 + [self.OFF] * 8) # off after 20 of 60 minutes
|
||||
self.assertEqual([verdict(early, h) for h in (15, 30, 60)], ['matched', 'counterfactual', 'counterfactual'])
|
||||
brief = ledger_with([0, 0, 0, 0], [self.OFF] * 4 + [240.0] + [self.OFF] * 7) # 4 minutes on
|
||||
self.assertEqual([verdict(brief, h) for h in (15, 30)], ['matched', 'approximate'])
|
||||
long = ledger_with([0, 0, 0, 0], [self.OFF] * 4 + [self.ON, self.ON] + [self.OFF] * 6)
|
||||
self.assertEqual(verdict(long, 30), 'counterfactual')
|
||||
|
||||
def test_several_cycles_followed_closely_match(self):
|
||||
plan = [1, 0, 1, 0, 1, 0, 1, 0]
|
||||
relay = []
|
||||
for block in plan:
|
||||
relay += [self.ON] * 3 if block else [30.0, self.OFF, self.OFF] # each off 30 s late
|
||||
ledger = ledger_with(plan, relay)
|
||||
self.assertEqual(ledger.deviation({'time': T0, 'plan': plan}, 120)[0], 120.0) # four transitions
|
||||
self.assertEqual(verdict(ledger, 60), 'matched')
|
||||
self.assertEqual(verdict(ledger, 120), 'matched')
|
||||
|
||||
def test_missing_relay_history_is_incomplete_not_matched(self):
|
||||
ledger = ledger_with([0, 0, 0, 0], [self.OFF, None] + [self.OFF] * 10)
|
||||
self.assertEqual([verdict(ledger, h) for h in (15, 60)], ['incomplete:relay_history'] * 2)
|
||||
# A plan already contradicted is counterfactual whatever the gap hides.
|
||||
ledger = ledger_with([1, 1, 1, 1], [None] + [self.OFF] * 11)
|
||||
self.assertEqual(verdict(ledger, 30), 'counterfactual')
|
||||
|
||||
def test_thermostat_disagreeing_is_counterfactual_never_an_error(self):
|
||||
ledger = ledger_with([1] * 4, [self.OFF] * 12, temps=lambda k: 20.0)
|
||||
stats = ledger.summary(['r'])
|
||||
self.assertEqual([stats[h]['counterfactual'] for h in (15, 30, 60)], [1, 1, 1])
|
||||
self.assertEqual([stats[h]['samples'] for h in (15, 30, 60)], [0, 0, 0])
|
||||
|
||||
def test_forecasts_made_with_assumed_inputs_are_incomplete(self):
|
||||
ledger = ledger_with([0] * 4, [self.OFF] * 12, uncertain=['window: Bedroom'])
|
||||
self.assertEqual(verdict(ledger, 15), 'incomplete:uncertain_inputs')
|
||||
self.assertEqual(ledger.summary(['r'])[15]['samples'], 0)
|
||||
|
||||
def test_counts_over_several_forecasts(self):
|
||||
ledger = ev.OperationalForecastLedger()
|
||||
ledger.observe(T0, {'r': 20.0}, None, None)
|
||||
for k, plan in enumerate(([0, 0], [1, 1], [0, 0])): # one forecast per poll, relay always off
|
||||
ledger.record(T0 + 300 * k, {'r': 20.0}, {'r': [20.0, 20.0]}, plan, 'rev1', {})
|
||||
ledger.observe(T0 + 300 * (k + 1), {'r': 20.0}, 0.0, 300.0)
|
||||
for k in range(4, 12):
|
||||
ledger.observe(T0 + 300 * k, {'r': 20.0}, 0.0, 300.0)
|
||||
stats = ledger.summary(['r'])[15]
|
||||
self.assertEqual((stats['matched'], stats['counterfactual'], stats['samples'], stats['incomplete']),
|
||||
(2, 1, 2, 0))
|
||||
restored = ev.OperationalForecastLedger(ledger.as_dict())
|
||||
self.assertEqual(restored.summary(['r']), ledger.summary(['r']))
|
||||
Reference in new issue
Block a user