buzeraandClaude Opus 5.5
2bddc74d37
Confirmed: when an open window's contact sensor became unknown or unavailable, the tracker ended the ventilation event as a close at the poll time, started recovery at full weight and forecasts assumed the window closed. (Learning was already excluded for that zone.) Each window is now OPEN, CLOSED or UNKNOWN with its last known state, since when, and since when it is unknown: - an unknown sensor keeps an open window's event (and the zone's "open" status) going; forecasts use the last known state; - a close observed only after an outage is recorded as unconfirmed: closed=None with the reported time and a maximum duration; recovery is assumed from that report but not used for any learning; - opening seen only after an outage starts an event without a temperature-before, so its recovery is not learned either; - events count minutes with an unknown sensor; Status shows each window's last known state and unknown minutes; - one unknown sensor only blocks learning for its own zone. Predictions with an unavailable wind sensor now keep an open window's natural ventilation (calm air) instead of dropping the term; learning still refuses unknown inputs. Regression tests for every OPEN/CLOSED/UNKNOWN transition, a six-hour outage, multiple windows with one unknown, and missing wind during ventilation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
235 lines
11 KiB
Python
235 lines
11 KiB
Python
"""Walk-forward evaluation of the thermal model against what actually happened.
|
|
|
|
Pure Python; no Home Assistant imports.
|
|
|
|
At every poll the readings, the outdoor temperature, the radiator-lag estimate
|
|
and each room's model parameters are recorded. When 15/30/60/120 minutes have
|
|
passed, each room is replayed from that earlier poll with the parameters it had
|
|
then (so later data is never used), the outdoor temperature held at its starting
|
|
value, and the radiator lag that actually followed from the relay's real ON/OFF
|
|
history. The replayed temperature is compared with the reading now.
|
|
|
|
This scores the thermal model, not the MPC's proposed plan: Generic Thermostat
|
|
controls the relay, so the plan is never executed. Two baselines are scored on
|
|
the same spans: persistence (no change) and the base 3-parameter model. The full
|
|
model replays all zones together with the electrical heat, neighbour exchange and
|
|
window openings that actually happened, so its improvement over the base model
|
|
shows whether those features earn their keep.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections import deque
|
|
from datetime import datetime, timedelta
|
|
from math import isfinite
|
|
from typing import Any
|
|
|
|
from .physics import Learner
|
|
from .windows import ventilation_vector_for_prediction
|
|
from .zone_model import feature_vector
|
|
|
|
HORIZONS = (15, 30, 60, 120) # minutes
|
|
WINDOW_DAYS = 7
|
|
STEP_SECONDS = 300
|
|
|
|
|
|
def replay(temp: float, outside: float, theta: tuple[float, float, float],
|
|
steps: list[tuple[float, float]]) -> float:
|
|
"""Advance one room through (end-of-step lag, step seconds) with fixed parameters."""
|
|
model = Learner(theta=list(theta))
|
|
for lag, seconds in steps:
|
|
# Same 5-minute model and jitter normalization as learning.
|
|
temp += model.predict_delta(temp, outside, lag) * seconds / STEP_SECONDS
|
|
return temp
|
|
|
|
|
|
def _dot(theta: tuple[float, ...], x: list[float]) -> float:
|
|
return sum(t * v for t, v in zip(theta, x))
|
|
|
|
|
|
def model_snapshot(model: Any) -> dict[str, Any]:
|
|
"""Frozen parameters of a ZoneModel, so later learning never leaks into a replay."""
|
|
def layer(learner):
|
|
return (list(learner.features), tuple(learner.theta)) if learner is not None else None
|
|
return {"base": tuple(model.base.theta), "uses_extended": model.uses_extended,
|
|
"extended": layer(model.extended), "ventilation": layer(model.ventilation)}
|
|
|
|
|
|
def snapshot_delta(snapshot: dict[str, Any], temp: float, inputs: dict[str, Any]) -> float:
|
|
"""5-minute change predicted by a model snapshot (see zone_model.ZoneModel.delta)."""
|
|
main = None
|
|
if snapshot["uses_extended"] and snapshot["extended"]:
|
|
features, theta = snapshot["extended"]
|
|
x = feature_vector(features, temp, inputs)
|
|
main = _dot(theta, x) if x is not None else None
|
|
if main is None:
|
|
a, b, c = snapshot["base"]
|
|
main = a * (inputs["outside"] - temp) + b * inputs["lag"] + c
|
|
ventilation = inputs.get("ventilation")
|
|
if snapshot["ventilation"] and ventilation:
|
|
features, theta = snapshot["ventilation"]
|
|
x = ventilation_vector_for_prediction(features, temp, inputs["outside"], ventilation)
|
|
if x is not None:
|
|
main += _dot(theta, x)
|
|
return main
|
|
|
|
|
|
def replay_zones(origin: dict[str, Any], path: list[dict[str, Any]]) -> dict[str, float]:
|
|
"""All zones advanced together with the origin's models and the actual inputs."""
|
|
state = dict(origin["temps"])
|
|
for before, poll in zip(path, path[1:]):
|
|
seconds = (poll["time"] - before["time"]).total_seconds()
|
|
new = {}
|
|
for zone, temp in state.items():
|
|
actual = poll["inputs"].get(zone, {})
|
|
inputs = {"outside": origin["outside"], "lag": poll["lag"], "temps": state,
|
|
"internal": actual.get("internal"), "ventilation": actual.get("ventilation")}
|
|
new[zone] = temp + snapshot_delta(origin["models"][zone], temp, inputs) * seconds / STEP_SECONDS
|
|
state = new
|
|
return state
|
|
|
|
|
|
class ForecastEvaluator:
|
|
"""Score replayed forecasts per room and horizon over the last WINDOW_DAYS days."""
|
|
|
|
def __init__(self, max_lag_uncertainty: float) -> None:
|
|
self._max_lag_uncertainty = max_lag_uncertainty
|
|
self._history: deque[dict[str, Any]] = deque(maxlen=HORIZONS[-1] * 60 // STEP_SECONDS + 1)
|
|
# UTC day -> horizon -> room ->
|
|
# [count, sum error, sum |error|, sum |persistence error|, sum |base model error|]
|
|
self._days: dict[str, dict[str, dict[str, list[float]]]] = {}
|
|
|
|
def reset_window(self) -> None:
|
|
"""Inputs were missing: no forecast may be scored across this poll."""
|
|
self._history.clear()
|
|
|
|
def observe(self, time: datetime, temps: dict[str, float], outside: float, lag: float,
|
|
lag_uncertainty: float, theta: dict[str, tuple[float, float, float]],
|
|
models: dict[str, dict[str, Any]] | None = None,
|
|
inputs: dict[str, dict[str, Any]] | None = None) -> None:
|
|
"""Record a poll, then score every forecast that has reached its horizon.
|
|
|
|
`models`: model_snapshot per zone (base-only when omitted); `inputs`: this
|
|
poll's actual electrical heat and ventilation inputs per zone.
|
|
"""
|
|
if self._history and not 270 <= (time - self._history[-1]["time"]).total_seconds() <= 330:
|
|
self._history.clear() # irregular poll spacing: replays would be unreliable
|
|
models = models or {}
|
|
snapshots = {zone: models.get(zone) or {"base": tuple(theta[zone]), "uses_extended": False,
|
|
"extended": None, "ventilation": None}
|
|
for zone in temps if zone in theta}
|
|
self._history.append({"time": time, "temps": dict(temps), "outside": outside, "lag": lag,
|
|
"lag_uncertainty": lag_uncertainty, "theta": dict(theta),
|
|
"models": snapshots, "inputs": dict(inputs or {})})
|
|
self._score()
|
|
self._prune(time)
|
|
|
|
def _score(self) -> None:
|
|
history = list(self._history)
|
|
latest = history[-1]
|
|
day = latest["time"].date().isoformat()
|
|
for horizon in HORIZONS:
|
|
steps = horizon * 60 // STEP_SECONDS
|
|
if len(history) < steps + 1:
|
|
continue
|
|
path = history[-(steps + 1):]
|
|
# The actual heat input must be known along the way.
|
|
if any(poll["lag_uncertainty"] > self._max_lag_uncertainty for poll in path[1:]):
|
|
continue
|
|
origin = path[0]
|
|
inputs = [(poll["lag"], (poll["time"] - before["time"]).total_seconds())
|
|
for before, poll in zip(path, path[1:])]
|
|
full = replay_zones(origin, path)
|
|
for room, actual in latest["temps"].items():
|
|
start, theta = origin["temps"].get(room), origin["theta"].get(room)
|
|
if start is None or theta is None:
|
|
continue
|
|
error = full[room] - actual
|
|
base_error = replay(start, origin["outside"], theta, inputs) - actual
|
|
bucket = self._days.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
|
|
room, [0, 0.0, 0.0, 0.0, 0.0])
|
|
bucket[0] += 1
|
|
bucket[1] += error
|
|
bucket[2] += abs(error)
|
|
bucket[3] += abs(start - actual)
|
|
bucket[4] += abs(base_error)
|
|
|
|
def _prune(self, time: datetime) -> None:
|
|
oldest = (time.date() - timedelta(days=WINDOW_DAYS - 1)).isoformat()
|
|
for day in [day for day in self._days if day < oldest]:
|
|
del self._days[day]
|
|
|
|
def summary(self, rooms: list[str]) -> dict[int, dict[str, Any]]:
|
|
"""Error statistics per horizon, overall and per configured room."""
|
|
result = {}
|
|
for horizon in HORIZONS:
|
|
per_room: dict[str, list[float]] = {}
|
|
for day in self._days.values():
|
|
for room, values in day.get(str(horizon), {}).items():
|
|
if room in rooms:
|
|
totals = per_room.setdefault(room, [0, 0.0, 0.0, 0.0, 0.0])
|
|
for i, value in enumerate(values):
|
|
totals[i] += value
|
|
overall = [sum(values[i] for values in per_room.values()) for i in range(5)]
|
|
result[horizon] = {**_stats(overall),
|
|
"rooms": {room: _stats(values) for room, values in per_room.items()}}
|
|
return result
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
# Rounded copy: shorter JSON on every save; in-memory sums stay exact.
|
|
return {"days": {day: {horizon: {room: [v[0], *(round(x, 6) for x in v[1:])]
|
|
for room, v in by_room.items()}
|
|
for horizon, by_room in by_horizon.items()}
|
|
for day, by_horizon in self._days.items()}}
|
|
|
|
def load(self, data: Any) -> None:
|
|
"""Restore stored statistics, keeping only well-formed entries."""
|
|
self._days = {}
|
|
days = data.get("days") if isinstance(data, dict) else None
|
|
if not isinstance(days, dict):
|
|
return
|
|
horizons = {str(horizon) for horizon in HORIZONS}
|
|
for day, by_horizon in days.items():
|
|
try:
|
|
datetime.strptime(day, "%Y-%m-%d")
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if not isinstance(by_horizon, dict):
|
|
continue
|
|
for horizon, by_room in by_horizon.items():
|
|
if horizon not in horizons or not isinstance(by_room, dict):
|
|
continue
|
|
for room, values in by_room.items():
|
|
if _valid_bucket(room, values):
|
|
# 0.3 buckets had no base column: the model then was the base model.
|
|
base = values[4] if len(values) == 5 else values[2]
|
|
self._days.setdefault(day, {}).setdefault(horizon, {})[room] = [
|
|
values[0], float(values[1]), float(values[2]), float(values[3]), float(base)]
|
|
|
|
|
|
def _valid_bucket(room: Any, values: Any) -> bool:
|
|
if not isinstance(room, str) or not isinstance(values, list) or len(values) not in (4, 5):
|
|
return False
|
|
count, *sums = values
|
|
return (type(count) is int and count >= 0
|
|
and all(type(v) in (int, float) and isfinite(v) for v in sums)
|
|
and all(v >= 0 for v in sums[1:]))
|
|
|
|
|
|
def _stats(values: list[float]) -> dict[str, Any]:
|
|
count, error, absolute, persistence, base = values
|
|
if not count:
|
|
return {"samples": 0, "mae": None, "bias": None, "persistence_mae": None, "skill": None,
|
|
"base_model_mae": None, "improvement_vs_base": None}
|
|
return {
|
|
"samples": int(count),
|
|
"mae": round(absolute / count, 3),
|
|
"bias": round(error / count, 3),
|
|
"persistence_mae": round(persistence / count, 3),
|
|
# Share of the persistence error removed by the model (1 = perfect, < 0 = worse).
|
|
"skill": round(1 - absolute / persistence, 3) if persistence > 0 else None,
|
|
"base_model_mae": round(base / count, 3),
|
|
# Share of the base model's error removed by electrical heat, neighbours and
|
|
# ventilation (0 = no gain, < 0 = the extra features make it worse).
|
|
"improvement_vs_base": round(1 - absolute / base, 3) if base > 0 else None,
|
|
}
|