heating-mpc-ha/custom_components/heating_mpc/evaluation.py

"""Walk-forward evaluation of the thermal model against what actually happened.

Pure Python; no Home Assistant imports.

At every poll the readings, the outdoor temperature, the radiator-lag estimate
and each room's model parameters are recorded. When 15/30/60/120 minutes have
passed, each room is replayed from that earlier poll with the parameters it had
then (so later data is never used), the outdoor temperature held at its starting
value, and the radiator lag that actually followed from the relay's real ON/OFF
history. The replayed temperature is compared with the reading now.

This scores the thermal model, not the MPC's proposed plan: Generic Thermostat
controls the relay, so the plan is never executed. Two baselines are scored on
the same spans: persistence (no change) and the base 3-parameter model. The full
model replays all zones together with the electrical heat, neighbour exchange and
window openings that actually happened, so its improvement over the base model
shows whether those features earn their keep.
"""
from __future__ import annotations

from collections import deque
from datetime import datetime, timedelta
from math import isfinite
from typing import Any

from .physics import Learner
from .windows import ventilation_vector_for_prediction
from .zone_model import feature_vector

HORIZONS = (15, 30, 60, 120)  # minutes
WINDOW_DAYS = 7
STEP_SECONDS = 300


def replay(temp: float, outside: float, theta: tuple[float, float, float],
           steps: list[tuple[float, float]]) -> float:
    """Advance one room through (end-of-step lag, step seconds) with fixed parameters."""
    model = Learner(theta=list(theta))
    for lag, seconds in steps:
        # Same 5-minute model and jitter normalization as learning.
        temp += model.predict_delta(temp, outside, lag) * seconds / STEP_SECONDS
    return temp


def _dot(theta: tuple[float, ...], x: list[float]) -> float:
    return sum(t * v for t, v in zip(theta, x))


def model_snapshot(model: Any) -> dict[str, Any]:
    """Frozen parameters of a ZoneModel, so later learning never leaks into a replay."""
    def layer(learner):
        return (list(learner.features), tuple(learner.theta)) if learner is not None else None
    return {"base": tuple(model.base.theta), "uses_extended": model.uses_extended,
            "extended": layer(model.extended), "ventilation": layer(model.ventilation)}


def snapshot_delta(snapshot: dict[str, Any], temp: float, inputs: dict[str, Any]) -> float:
    """5-minute change predicted by a model snapshot (see zone_model.ZoneModel.delta)."""
    main = None
    if snapshot["uses_extended"] and snapshot["extended"]:
        features, theta = snapshot["extended"]
        x = feature_vector(features, temp, inputs)
        main = _dot(theta, x) if x is not None else None
    if main is None:
        a, b, c = snapshot["base"]
        main = a * (inputs["outside"] - temp) + b * inputs["lag"] + c
    ventilation = inputs.get("ventilation")
    if snapshot["ventilation"] and ventilation:
        features, theta = snapshot["ventilation"]
        x = ventilation_vector_for_prediction(features, temp, inputs["outside"], ventilation)
        if x is not None:
            main += _dot(theta, x)
    return main


def replay_zones(origin: dict[str, Any], path: list[dict[str, Any]]) -> dict[str, float]:
    """All zones advanced together with the origin's models and the actual inputs."""
    state = dict(origin["temps"])
    for before, poll in zip(path, path[1:]):
        seconds = (poll["time"] - before["time"]).total_seconds()
        new = {}
        for zone, temp in state.items():
            actual = poll["inputs"].get(zone, {})
            inputs = {"outside": origin["outside"], "lag": poll["lag"], "temps": state,
                      "internal": actual.get("internal"), "ventilation": actual.get("ventilation")}
            new[zone] = temp + snapshot_delta(origin["models"][zone], temp, inputs) * seconds / STEP_SECONDS
        state = new
    return state


class ForecastEvaluator:
    """Score replayed forecasts per room and horizon over the last WINDOW_DAYS days."""

    def __init__(self, max_lag_uncertainty: float) -> None:
        self._max_lag_uncertainty = max_lag_uncertainty
        self._history: deque[dict[str, Any]] = deque(maxlen=HORIZONS[-1] * 60 // STEP_SECONDS + 1)
        # UTC day -> horizon -> room ->
        #   [count, sum error, sum |error|, sum |persistence error|, sum |base model error|]
        self._days: dict[str, dict[str, dict[str, list[float]]]] = {}

    def reset_window(self) -> None:
        """Inputs were missing: no forecast may be scored across this poll."""
        self._history.clear()

    def observe(self, time: datetime, temps: dict[str, float], outside: float, lag: float,
                lag_uncertainty: float, theta: dict[str, tuple[float, float, float]],
                models: dict[str, dict[str, Any]] | None = None,
                inputs: dict[str, dict[str, Any]] | None = None) -> None:
        """Record a poll, then score every forecast that has reached its horizon.

        `models`: model_snapshot per zone (base-only when omitted); `inputs`: this
        poll's actual electrical heat and ventilation inputs per zone.
        """
        if self._history and not 270 <= (time - self._history[-1]["time"]).total_seconds() <= 330:
            self._history.clear()  # irregular poll spacing: replays would be unreliable
        models = models or {}
        snapshots = {zone: models.get(zone) or {"base": tuple(theta[zone]), "uses_extended": False,
                                               "extended": None, "ventilation": None}
                     for zone in temps if zone in theta}
        self._history.append({"time": time, "temps": dict(temps), "outside": outside, "lag": lag,
                              "lag_uncertainty": lag_uncertainty, "theta": dict(theta),
                              "models": snapshots, "inputs": dict(inputs or {})})
        self._score()
        self._prune(time)

    def _score(self) -> None:
        history = list(self._history)
        latest = history[-1]
        day = latest["time"].date().isoformat()
        for horizon in HORIZONS:
            steps = horizon * 60 // STEP_SECONDS
            if len(history) < steps + 1:
                continue
            path = history[-(steps + 1):]
            # The actual heat input must be known along the way.
            if any(poll["lag_uncertainty"] > self._max_lag_uncertainty for poll in path[1:]):
                continue
            origin = path[0]
            inputs = [(poll["lag"], (poll["time"] - before["time"]).total_seconds())
                      for before, poll in zip(path, path[1:])]
            full = replay_zones(origin, path)
            for room, actual in latest["temps"].items():
                start, theta = origin["temps"].get(room), origin["theta"].get(room)
                if start is None or theta is None:
                    continue
                error = full[room] - actual
                base_error = replay(start, origin["outside"], theta, inputs) - actual
                bucket = self._days.setdefault(day, {}).setdefault(str(horizon), {}).setdefault(
                    room, [0, 0.0, 0.0, 0.0, 0.0])
                bucket[0] += 1
                bucket[1] += error
                bucket[2] += abs(error)
                bucket[3] += abs(start - actual)
                bucket[4] += abs(base_error)

    def _prune(self, time: datetime) -> None:
        oldest = (time.date() - timedelta(days=WINDOW_DAYS - 1)).isoformat()
        for day in [day for day in self._days if day < oldest]:
            del self._days[day]

    def summary(self, rooms: list[str]) -> dict[int, dict[str, Any]]:
        """Error statistics per horizon, overall and per configured room."""
        result = {}
        for horizon in HORIZONS:
            per_room: dict[str, list[float]] = {}
            for day in self._days.values():
                for room, values in day.get(str(horizon), {}).items():
                    if room in rooms:
                        totals = per_room.setdefault(room, [0, 0.0, 0.0, 0.0, 0.0])
                        for i, value in enumerate(values):
                            totals[i] += value
            overall = [sum(values[i] for values in per_room.values()) for i in range(5)]
            result[horizon] = {**_stats(overall),
                               "rooms": {room: _stats(values) for room, values in per_room.items()}}
        return result

    def forget(self, room: str) -> None:
        """Drop a zone's statistics, for example when its temperature is measured differently."""
        for by_horizon in self._days.values():
            for by_room in by_horizon.values():
                by_room.pop(room, None)
        for poll in self._history:
            poll["temps"].pop(room, None)

    def as_dict(self) -> dict[str, Any]:
        # Rounded copy: shorter JSON on every save; in-memory sums stay exact.
        return {"days": {day: {horizon: {room: [v[0], *(round(x, 6) for x in v[1:])]
                                         for room, v in by_room.items()}
                               for horizon, by_room in by_horizon.items()}
                         for day, by_horizon in self._days.items()}}

    def load(self, data: Any) -> None:
        """Restore stored statistics, keeping only well-formed entries."""
        self._days = {}
        days = data.get("days") if isinstance(data, dict) else None
        if not isinstance(days, dict):
            return
        horizons = {str(horizon) for horizon in HORIZONS}
        for day, by_horizon in days.items():
            try:
                datetime.strptime(day, "%Y-%m-%d")
            except (TypeError, ValueError):
                continue
            if not isinstance(by_horizon, dict):
                continue
            for horizon, by_room in by_horizon.items():
                if horizon not in horizons or not isinstance(by_room, dict):
                    continue
                for room, values in by_room.items():
                    if _valid_bucket(room, values):
                        # 0.3 buckets had no base column: the model then was the base model.
                        base = values[4] if len(values) == 5 else values[2]
                        self._days.setdefault(day, {}).setdefault(horizon, {})[room] = [
                            values[0], float(values[1]), float(values[2]), float(values[3]), float(base)]


def _valid_bucket(room: Any, values: Any) -> bool:
    if not isinstance(room, str) or not isinstance(values, list) or len(values) not in (4, 5):
        return False
    count, *sums = values
    return (type(count) is int and count >= 0
            and all(type(v) in (int, float) and isfinite(v) for v in sums)
            and all(v >= 0 for v in sums[1:]))


def _stats(values: list[float]) -> dict[str, Any]:
    count, error, absolute, persistence, base = values
    if not count:
        return {"samples": 0, "mae": None, "bias": None, "persistence_mae": None, "skill": None,
                "base_model_mae": None, "improvement_vs_base": None}
    return {
        "samples": int(count),
        "mae": round(absolute / count, 3),
        "bias": round(error / count, 3),
        "persistence_mae": round(persistence / count, 3),
        # Share of the persistence error removed by the model (1 = perfect, < 0 = worse).
        "skill": round(1 - absolute / persistence, 3) if persistence > 0 else None,
        "base_model_mae": round(base / count, 3),
        # Share of the base model's error removed by electrical heat, neighbours and
        # ventilation (0 = no gain, < 0 = the extra features make it worse).
        "improvement_vs_base": round(1 - absolute / base, 3) if base > 0 else None,
    }