buzeraandClaude Opus 5.5
b1577a0c78
Validation: the walk-forward evaluator now replays all zones together with the full model (electrical heat, neighbour exchange, ventilation and recovery) using the parameters frozen at the forecast start and the electrical heat and window openings that actually happened, and scores the base 3-parameter model on the same spans. Forecast Error sensors add base_model_mae and improvement_vs_base (share of the base model's error the new features remove; 0 = no gain, negative = worse). Saved 0.3 statistics load with their own error as the base column. On synthetic apartments, 60-minute error full vs base: gaming office 0.031 vs 0.182 °C (83 % better), bedroom with ventilation 0.026 vs 0.218 °C (88 %), two coupled rooms 0.024 vs 0.037 °C (34 %; gone by 120 min, where the constant-outdoor assumption dominates). The same ventilation week without window sensors shows no improvement, as it should. These check the method, not real-world accuracy. Performance: every term of a zone's 5-minute change is affine in the zone temperatures and the radiator lag, and which layer applies depends only on which inputs exist. compile_step_model reads the coefficients off the real model per zone and step, so the plan search does a few multiply-adds per step. It matches the direct model to 1e-12 and makes a 180-minute, 5-zone search 4.5x faster (0.73 s here, close to the old base-only search). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
235 lines
11 KiB
Python
235 lines
11 KiB
Python
"""Walk-forward evaluation of the thermal model against what actually happened.
|
|
|
|
Pure Python; no Home Assistant imports.
|
|
|
|
At every poll the readings, the outdoor temperature, the radiator-lag estimate
|
|
and each room's model parameters are recorded. When 15/30/60/120 minutes have
|
|
passed, each room is replayed from that earlier poll with the parameters it had
|
|
then (so later data is never used), the outdoor temperature held at its starting
|
|
value, and the radiator lag that actually followed from the relay's real ON/OFF
|
|
history. The replayed temperature is compared with the reading now.
|
|
|
|
This scores the thermal model, not the MPC's proposed plan: Generic Thermostat
|
|
controls the relay, so the plan is never executed. Two baselines are scored on
|
|
the same spans: persistence (no change) and the base 3-parameter model. The full
|
|
model replays all zones together with the electrical heat, neighbour exchange and
|
|
window openings that actually happened, so its improvement over the base model
|
|
shows whether those features earn their keep.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections import deque
|
|
from datetime import datetime, timedelta
|
|
from math import isfinite
|
|
from typing import Any
|
|
|
|
from .physics import Learner
|
|
from .windows import ventilation_vector
|
|
from .zone_model import feature_vector
|
|
|
|
HORIZONS = (15, 30, 60, 120) # minutes
|
|
WINDOW_DAYS = 7
|
|
STEP_SECONDS = 300
|
|
|
|
|
|
def replay(temp: float, outside: float, theta: tuple[float, float, float],
|
|
steps: list[tuple[float, float]]) -> float:
|
|
"""Advance one room through (end-of-step lag, step seconds) with fixed parameters."""
|
|
model = Learner(theta=list(theta))
|
|
for lag, seconds in steps:
|
|
# Same 5-minute model and jitter normalization as learning.
|
|
temp += model.predict_delta(temp, outside, lag) * seconds / STEP_SECONDS
|
|
return temp
|
|
|
|
|
|
def _dot(theta: tuple[float, ...], x: list[float]) -> float:
|
|
return sum(t * v for t, v in zip(theta, x))
|
|
|
|
|
|
def model_snapshot(model: Any) -> dict[str, Any]:
|
|
"""Frozen parameters of a ZoneModel, so later learning never leaks into a replay."""
|
|
def layer(learner):
|
|
return (list(learner.features), tuple(learner.theta)) if learner is not None else None
|
|
return {"base": tuple(model.base.theta), "uses_extended": model.uses_extended,
|
|
"extended": layer(model.extended), "ventilation": layer(model.ventilation)}
|
|
|
|
|
|
def snapshot_delta(snapshot: dict[str, Any], temp: float, inputs: dict[str, Any]) -> float:
|
|
"""5-minute change predicted by a model snapshot (see zone_model.ZoneModel.delta)."""
|
|
main = None
|
|
if snapshot["uses_extended"] and snapshot["extended"]:
|
|
features, theta = snapshot["extended"]
|
|
x = feature_vector(features, temp, inputs)
|
|
main = _dot(theta, x) if x is not None else None
|
|
if main is None:
|
|
a, b, c = snapshot["base"]
|
|
main = a * (inputs["outside"] - temp) + b * inputs["lag"] + c
|
|
ventilation = inputs.get("ventilation")
|
|
if snapshot["ventilation"] and ventilation:
|
|
features, theta = snapshot["ventilation"]
|
|
x = ventilation_vector(features, temp, inputs["outside"], ventilation)
|
|
if x is not None:
|
|
main += _dot(theta, x)
|
|
return main
|
|
|
|
|
|
def replay_zones(origin: dict[str, Any], path: list[dict[str, Any]]) -> dict[str, float]:
|
|
"""All zones advanced together with the origin's models and the actual inputs."""
|
|
state = dict(origin["temps"])
|
|
for before, poll in zip(path, path[1:]):
|
|
seconds = (poll["time"] - before["time"]).total_seconds()
|
|
new = {}
|
|
for zone, temp in state.items():
|
|
actual = poll["inputs"].get(zone, {})
|
|
inputs = {"outside": origin["outside"], "lag": poll["lag"], "temps": state,
|
|
"internal": actual.get("internal"), "ventilation": actual.get("ventilation")}
|
|
new[zone] = temp + snapshot_delta(origin["models"][zone], temp, inputs) * seconds / STEP_SECONDS
|
|
state = new
|
|
return state
|
|
|
|
|
|
class ForecastEvaluator:
|
|
"""Score replayed forecasts per room and horizon over the last WINDOW_DAYS days."""
|
|
|
|
def __init__(self, max_lag_uncertainty: float) -> None:
|
|
self._max_lag_uncertainty = max_lag_uncertainty
|
|
self._history: deque[dict[str, Any]] = deque(maxlen=HORIZONS[-1] * 60 // STEP_SECONDS + 1)
|
|
# UTC day -> horizon -> room ->
|
|
# [count, sum error, sum |error|, sum |persistence error|, sum |base model error|]
|
|
self._days: dict[str, dict[str, dict[str, list[float]]]] = {}
|
|
|
|
def reset_window(self) -> None:
|
|
"""Inputs were missing: no forecast may be scored across this poll."""
|
|
self._history.clear()
|
|
|
|
def observe(self, time: datetime, temps: dict[str, float], outside: float, lag: float,
|
|
lag_uncertainty: float, theta: dict[str, tuple[float, float, float]],
|
|
models: dict[str, dict[str, Any]] | None = None,
|
|
inputs: dict[str, dict[str, Any]] | None = None) -> None:
|
|
"""Record a poll, then score every forecast that has reached its horizon.
|
|
|
|
`models`: model_snapshot per zone (base-only when omitted); `inputs`: this
|
|
poll's actual electrical heat and ventilation inputs per zone.
|
|
"""
|
|
if self._history and not 270 <= (time - self._history[-1]["time"]).total_seconds() <= 330:
|
|
self._history.clear() # irregular poll spacing: replays would be unreliable
|
|
models = models or {}
|
|
snapshots = {zone: models.get(zone) or {"base": tuple(theta[zone]), "uses_extended": False,
|
|
"extended": None, "ventilation": None}
|
|
for zone in temps if zone in theta}
|
|
self._history.append({"time": time, "temps": dict(temps), "outside": outside, "lag": lag,
|
|
"lag_uncertainty": lag_uncertainty, "theta": dict(theta),
|
|
"models": snapshots, "inputs": dict(inputs or {})})
|
|
self._score()
|
|
self._prune(time)
|
|
|
|
def _score(self) -> None:
|
|
history = list(self._history)
|
|
latest = history[-1]
|
|
day = latest["time"].date().isoformat()
|
|
for horizon in HORIZONS:
|
|
steps = horizon * 60 // STEP_SECONDS
|
|
if len(history) < steps + 1:
|
|
continue
|
|
path = history[-(steps + 1):]
|
|
# The actual heat input must be known along the way.
|
|
if any(poll["lag_uncertainty"] > self._max_lag_uncertainty for poll in path[1:]):
|
|
continue
|
|
origin = path[0]
|
|
inputs = [(poll["lag"], (poll["time"] - before["time"]).total_seconds())
|
|
for before, poll in zip(path, path[1:])]
|
|
full = replay_zones(origin, path)
|
|
for room, actual in latest["temps"].items():
|
|
start, theta = origin["temps"].get(room), origin["theta"].get(room)
|
|
if start is None or theta is None:
|
|
continue
|
|
error = full[room] - actual
|
|
base_error = replay(start, origin["outside"], theta, inputs) - actual
|
|
bucket = self._days.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
|
|
room, [0, 0.0, 0.0, 0.0, 0.0])
|
|
bucket[0] += 1
|
|
bucket[1] += error
|
|
bucket[2] += abs(error)
|
|
bucket[3] += abs(start - actual)
|
|
bucket[4] += abs(base_error)
|
|
|
|
def _prune(self, time: datetime) -> None:
|
|
oldest = (time.date() - timedelta(days=WINDOW_DAYS - 1)).isoformat()
|
|
for day in [day for day in self._days if day < oldest]:
|
|
del self._days[day]
|
|
|
|
def summary(self, rooms: list[str]) -> dict[int, dict[str, Any]]:
|
|
"""Error statistics per horizon, overall and per configured room."""
|
|
result = {}
|
|
for horizon in HORIZONS:
|
|
per_room: dict[str, list[float]] = {}
|
|
for day in self._days.values():
|
|
for room, values in day.get(str(horizon), {}).items():
|
|
if room in rooms:
|
|
totals = per_room.setdefault(room, [0, 0.0, 0.0, 0.0, 0.0])
|
|
for i, value in enumerate(values):
|
|
totals[i] += value
|
|
overall = [sum(values[i] for values in per_room.values()) for i in range(5)]
|
|
result[horizon] = {**_stats(overall),
|
|
"rooms": {room: _stats(values) for room, values in per_room.items()}}
|
|
return result
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
# Rounded copy: shorter JSON on every save; in-memory sums stay exact.
|
|
return {"days": {day: {horizon: {room: [v[0], *(round(x, 6) for x in v[1:])]
|
|
for room, v in by_room.items()}
|
|
for horizon, by_room in by_horizon.items()}
|
|
for day, by_horizon in self._days.items()}}
|
|
|
|
def load(self, data: Any) -> None:
|
|
"""Restore stored statistics, keeping only well-formed entries."""
|
|
self._days = {}
|
|
days = data.get("days") if isinstance(data, dict) else None
|
|
if not isinstance(days, dict):
|
|
return
|
|
horizons = {str(horizon) for horizon in HORIZONS}
|
|
for day, by_horizon in days.items():
|
|
try:
|
|
datetime.strptime(day, "%Y-%m-%d")
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if not isinstance(by_horizon, dict):
|
|
continue
|
|
for horizon, by_room in by_horizon.items():
|
|
if horizon not in horizons or not isinstance(by_room, dict):
|
|
continue
|
|
for room, values in by_room.items():
|
|
if _valid_bucket(room, values):
|
|
# 0.3 buckets had no base column: the model then was the base model.
|
|
base = values[4] if len(values) == 5 else values[2]
|
|
self._days.setdefault(day, {}).setdefault(horizon, {})[room] = [
|
|
values[0], float(values[1]), float(values[2]), float(values[3]), float(base)]
|
|
|
|
|
|
def _valid_bucket(room: Any, values: Any) -> bool:
|
|
if not isinstance(room, str) or not isinstance(values, list) or len(values) not in (4, 5):
|
|
return False
|
|
count, *sums = values
|
|
return (type(count) is int and count >= 0
|
|
and all(type(v) in (int, float) and isfinite(v) for v in sums)
|
|
and all(v >= 0 for v in sums[1:]))
|
|
|
|
|
|
def _stats(values: list[float]) -> dict[str, Any]:
|
|
count, error, absolute, persistence, base = values
|
|
if not count:
|
|
return {"samples": 0, "mae": None, "bias": None, "persistence_mae": None, "skill": None,
|
|
"base_model_mae": None, "improvement_vs_base": None}
|
|
return {
|
|
"samples": int(count),
|
|
"mae": round(absolute / count, 3),
|
|
"bias": round(error / count, 3),
|
|
"persistence_mae": round(persistence / count, 3),
|
|
# Share of the persistence error removed by the model (1 = perfect, < 0 = worse).
|
|
"skill": round(1 - absolute / persistence, 3) if persistence > 0 else None,
|
|
"base_model_mae": round(base / count, 3),
|
|
# Share of the base model's error removed by electrical heat, neighbours and
|
|
# ventilation (0 = no gain, < 0 = the extra features make it worse).
|
|
"improvement_vs_base": round(1 - absolute / base, 3) if base > 0 else None,
|
|
}
|