From c771bc360ab1c7b26b28ca2d6acde6101612f52e Mon Sep 17 00:00:00 2001 From: Andrei Buzera Date: Mon, 26 Oct 2026 18:21:18 +0200 Subject: [PATCH] Add HA lifecycle tests and CI --- .gitea/workflows/tests.yml | 18 +++++ .gitignore | 6 ++ AUDIT.md | 18 +++++ README.md | 36 +++++++++ ROADMAP.md | 11 ++- pytest.ini | 4 + requirements-test.txt | 4 + scripts/check.sh | 11 +++ scripts/validate_package.py | 28 +++++++ tests/ha/conftest.py | 40 ++++++++++ tests/ha/test_config_flow.py | 58 ++++++++++++++ tests/ha/test_lifecycle.py | 148 +++++++++++++++++++++++++++++++++++ 12 files changed, 378 insertions(+), 4 deletions(-) create mode 100644 .gitea/workflows/tests.yml create mode 100644 .gitignore create mode 100644 pytest.ini create mode 100644 requirements-test.txt create mode 100644 scripts/check.sh create mode 100644 scripts/validate_package.py create mode 100644 tests/ha/conftest.py create mode 100644 tests/ha/test_config_flow.py create mode 100644 tests/ha/test_lifecycle.py diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml new file mode 100644 index 0000000..58ae30b --- /dev/null +++ b/.gitea/workflows/tests.yml @@ -0,0 +1,18 @@ +name: Tests +on: + push: + pull_request: + workflow_dispatch: +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: https://github.com/actions/checkout@v4 + - uses: https://github.com/actions/setup-python@v5 + with: + python-version: '3.13' + cache: pip + - name: Install test dependencies + run: python -m pip install -r requirements-test.txt + - name: Run checks + run: sh scripts/check.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4bb2a84 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.coverage +htmlcov/ +.venv/ diff --git a/AUDIT.md b/AUDIT.md index ee69ab2..cdb9be0 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -106,3 +106,21 @@ validation remain open. See README for compatibility and safe trial details. The suite now has 42 passing offline tests. Compilation with the temporary cache and `git diff --check` pass. The API basis is HA's [state timestamp documentation](https://www.home-assistant.io/docs/configuration/state_object/). + + +## Follow-up: HA Core harness and CI + +Added a separate `tests/ha` pytest suite using the real Home Assistant 2025.12.0 +package and matching fixtures, pinned for Python 3.13. All 14 tests pass locally, +alongside the existing 42 offline tests. The full `scripts/check.sh` checks and +`pip check` passed. The suite exercises the actual config manager, config/options +schemas, registry, scheduler and Store API, with mocked disk I/O and simulated +sensor/reference entities. A service-call guard covers every test. No actuator +calls were observed. Tests include cold setup with stored data/manual off, but not +a full HA process restart or HAOS hardware validation. + +Added a Forgejo/Gitea Actions workflow and lightweight package/translation checks. +Remote runner execution, a modern-HA version matrix, dedicated lint/type checks, +full process restart, partial platform failure and real disk-failure testing remain +open. The test harness needed a pycares 4.11.0 pin for compatibility with HA's DNS +dependency. Integration runtime dependencies and behavior are unchanged. diff --git a/README.md b/README.md index 33a60ae..dd6b2fc 100644 --- a/README.md +++ b/README.md @@ -126,3 +126,39 @@ history remain future work. These checks run on the existing five-minute poll. After backing up and upgrading, check report ages in Status, adjust limits in Options to match your sensors, and verify fallback/recovery with an unavailable source. Generic Thermostat remains the sole controller. No HAOS trial was performed. + + +## Developer checks and Forgejo/Gitea Actions + +The HA Core test suite targets **Home Assistant 2025.12.0 on Python 3.13**, +matching the minimum version in `hacs.json`. Run in a dedicated development +virtual environment; these test dependencies are not installed in the integration +and do not change its local, standard-library model inference. + +```bash +python3.13 -m venv .venv +. .venv/bin/activate +python -m pip install -r requirements-test.txt +sh scripts/check.sh +``` + +The check script runs dependency consistency checks, 42 offline tests, 14 tests +using the real HA Core/fixture package, compilation, lightweight package metadata +and translation checks, and diff whitespace checks. A compatible DNS dependency +is pinned because pycares 5 breaks the older HA dependency's import. The script +also disables a transitive HA test dependency's optional metadata download. + +HA tests cover setup, config/options forms, migration, reload and entity cleanup, +scheduled sampling and poll cancellation, stored model restoration/deletion, +missing inputs, manual off/presets and optimizer failure. A fixture rejects all +HA service calls throughout each test. HA Core runs locally with simulated input +entities and mocked storage I/O; this does not validate HAOS, physical devices, +disk failures or full process restarts. The original standalone unittest command +still works without installing HA. + +`.gitea/workflows/tests.yml` runs the same checks for pushes and pull requests on +an `ubuntu-latest` Actions runner. Enable repository Actions and configure a runner +with that label. Dependencies and action downloads need internet access in CI; +model inference does not. Remote CI execution has not yet been verified. Local +validation used macOS and Python 3.13; Linux runner validation remains necessary. +No runtime integration behavior or manifest version changed in this test commit. diff --git a/ROADMAP.md b/ROADMAP.md index f82ce32..f958b82 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -40,7 +40,7 @@ Current behavior: **shadow mode only**. `generic_thermostat` remains the one and **Objective:** Make the existing observer trustworthy before extending it. - [x] Perform a source audit of v0.2.1: pure model, optimizer, config/options, coordinator update timing, persistent model and entity lifecycle. -- [ ] Add Home Assistant test harness coverage for initial setup, update, restart/reload, options changes, missing sensors, `heat/off`, target changes, teardown and uninstall. +- [x] Add Home Assistant test harness coverage for initial setup, update, restart/reload, options changes, missing sensors, `heat/off`, target changes, teardown and uninstall. - [ ] Verify that **zero physical-device service calls** occur in observation mode, including restart and failed update cases. - [ ] Establish deterministic handling of 5-minute transitions, polling jitter, timestamp discontinuities, missing data and real switch transitions between samples. - [x] Add input freshness thresholds and diagnostics that identify exactly which source is missing/stale/invalid. @@ -57,14 +57,17 @@ Current behavior: **shadow mode only**. `generic_thermostat` remains the one and See [AUDIT.md](AUDIT.md). The source audit and focused repairs are complete; Milestone 0 acceptance is **not** complete. Offline boundary doubles cover setup, reload/unload, migration, removal, corrupt storage, missing inputs and manual off. -They are not a full HA test harness. Per-room accepted ON/OFF counts, additive +A follow-up now adds 14 real HA Core 2025.12.0 tests on Python 3.13, including +config/options flows, scheduling, registry cleanup and Store API calls. Disk I/O +and source entities are mocked; HAOS and full-process restart tests remain open. Per-room accepted ON/OFF counts, additive payload schema v2, conservative relay-transition rejection, jitter normalization, Fahrenheit targets and forecast input-change guards are implemented. Configurable room/weather report-age limits and per-source quality diagnostics are now implemented with offline coverage. Sensor alignment remains open. -Next implement a supported HA test harness/CI. Then improve event-based sampling and identity/persistence -migration tests. Only after those reliability checks proceed to Milestone 1's +The HA Core harness and Forgejo/Gitea workflow are now added and pass locally; +remote runner execution remains unverified. Next improve event-based sampling and +identity/persistence migration tests, then extend the HA version matrix. Only after those reliability checks proceed to Milestone 1's recorded forecasts and walk-forward evaluation. Live control remains unauthorized. ## Milestone 1 — Explainable forecasts and dashboards (target v0.3.0) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..0d8f293 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +testpaths = tests/ha +asyncio_mode = auto +asyncio_default_fixture_loop_scope = function diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..29e3dfa --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,4 @@ +# Pins Home Assistant 2025.12.0 and its matching fixtures; requires Python 3.13. +pytest-homeassistant-custom-component==0.13.298 +# aiodns 3.5 (HA's pin) uses APIs removed in pycares 5. +pycares==4.11.0 diff --git a/scripts/check.sh b/scripts/check.sh new file mode 100644 index 0000000..0708bdb --- /dev/null +++ b/scripts/check.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Run with Python 3.13 and requirements-test.txt installed. +set -eu +# A transitive HA test dependency otherwise fetches metadata on import. +export LITELLM_LOCAL_MODEL_COST_MAP=True +python -m pip check +python -m unittest discover -s tests -v +python -m pytest tests/ha -q +python -m compileall -q custom_components/heating_mpc +python scripts/validate_package.py +git diff --check diff --git a/scripts/validate_package.py b/scripts/validate_package.py new file mode 100644 index 0000000..e8b5127 --- /dev/null +++ b/scripts/validate_package.py @@ -0,0 +1,28 @@ +"""Small local packaging and translation checks; not a replacement for hassfest.""" +import json +from pathlib import Path +import subprocess + +root = Path(__file__).resolve().parents[1] +component = root / "custom_components" / "heating_mpc" +manifest = json.loads((component / "manifest.json").read_text()) +assert manifest["domain"] == "heating_mpc" +assert manifest["config_flow"] is True +assert manifest["iot_class"] == "local_polling" +assert not manifest["requirements"], "Model inference should remain dependency-free" +json.loads((root / "hacs.json").read_text()) + + +def keys(value, prefix=()): + if not isinstance(value, dict): + return {prefix} + return set().union(*(keys(child, (*prefix, key)) for key, child in value.items())) + + +strings = json.loads((component / "strings.json").read_text()) +for language in ("en", "ro"): + translated = json.loads((component / "translations" / f"{language}.json").read_text()) + assert keys(strings) == keys(translated), f"Translation keys differ: {language}" +tracked = subprocess.check_output(["git", "ls-files", "custom_components"], cwd=root, text=True).splitlines() +assert not any("__pycache__" in path or path.endswith((".pyc", ".pyo")) for path in tracked) +print("Package metadata, translations and tracked component files passed") diff --git a/tests/ha/conftest.py b/tests/ha/conftest.py new file mode 100644 index 0000000..70e49e7 --- /dev/null +++ b/tests/ha/conftest.py @@ -0,0 +1,40 @@ +"""Real Home Assistant fixtures, isolated from the offline boundary doubles.""" +from unittest.mock import patch + +import pytest +from homeassistant.core import ServiceRegistry +from pytest_homeassistant_custom_component.common import MockConfigEntry + + +@pytest.fixture(autouse=True) +def custom_integrations(enable_custom_integrations): + """Let HA discover custom_components/heating_mpc in this checkout.""" + + +@pytest.fixture(autouse=True) +def no_actuator_calls(): + """Reject every HA service call during all tested observer lifecycle paths.""" + with patch.object(ServiceRegistry, "async_call", side_effect=AssertionError( + "The shadow integration must never call HA services" + )) as calls: + yield calls + calls.assert_not_called() + + +@pytest.fixture +def entry(): + return MockConfigEntry( + domain="heating_mpc", version=2, unique_id="home_heating", + data={"climate_entity": "climate.reference", "heater_entity": "switch.boiler", + "room_entities": ["sensor.room"], + "outside_temperature_sources": ["sensor.outside"], + "weights": {"sensor.room": 1.0}, "horizon_minutes": 60}, + ) + + +@pytest.fixture +def inputs(hass): + hass.states.async_set("sensor.room", "20", {"unit_of_measurement": "°C"}) + hass.states.async_set("sensor.outside", "5", {"unit_of_measurement": "°C"}) + hass.states.async_set("switch.boiler", "off") + hass.states.async_set("climate.reference", "heat", {"temperature": 22, "preset_mode": "comfort"}) diff --git a/tests/ha/test_config_flow.py b/tests/ha/test_config_flow.py new file mode 100644 index 0000000..f2afe46 --- /dev/null +++ b/tests/ha/test_config_flow.py @@ -0,0 +1,58 @@ +"""Validate actual HA form schemas, options and entry migration.""" +from homeassistant.config_entries import SOURCE_USER +from homeassistant.data_entry_flow import FlowResultType +from pytest_homeassistant_custom_component.common import MockConfigEntry + +from custom_components.heating_mpc.const import DOMAIN + + +async def test_config_flow(hass, inputs): + result = await hass.config_entries.flow.async_init(DOMAIN, context={"source": SOURCE_USER}) + assert result["type"] is FlowResultType.FORM + main = {"climate_entity": "climate.reference", "heater_entity": "switch.boiler", + "room_entities": ["sensor.room"]} + validated = result["data_schema"](main) + assert validated["room_max_age_minutes"] == 30 + assert validated["weather_max_age_minutes"] == 120 + result = await hass.config_entries.flow.async_configure(result["flow_id"], validated) + assert result["step_id"] == "weather" + result = await hass.config_entries.flow.async_configure( + result["flow_id"], result["data_schema"]({"outside_temperature_sources": ["sensor.outside"]})) + assert result["step_id"] == "weather_extra" + result = await hass.config_entries.flow.async_configure(result["flow_id"], result["data_schema"]({})) + assert result["step_id"] == "weights" + result = await hass.config_entries.flow.async_configure(result["flow_id"], {"weight__sensor.room": 2}) + assert result["type"] is FlowResultType.CREATE_ENTRY + assert result["data"]["weights"] == {"sensor.room": 2} + await hass.async_block_till_done() + + +async def test_options_flow_keeps_settings(hass, entry, inputs): + entry.add_to_hass(hass) + assert await hass.config_entries.async_setup(entry.entry_id) + await hass.async_block_till_done() + result = await hass.config_entries.options.async_init(entry.entry_id) + main = result["data_schema"]({"climate_entity": "climate.reference", + "heater_entity": "switch.boiler", "room_entities": ["sensor.room"], + "room_max_age_minutes": 45}) + result = await hass.config_entries.options.async_configure(result["flow_id"], main) + result = await hass.config_entries.options.async_configure( + result["flow_id"], result["data_schema"]({"outside_temperature_sources": ["sensor.outside"]})) + result = await hass.config_entries.options.async_configure(result["flow_id"], result["data_schema"]({})) + result = await hass.config_entries.options.async_configure(result["flow_id"], {"weight__sensor.room": 3}) + assert result["type"] is FlowResultType.CREATE_ENTRY + await hass.async_block_till_done() + assert entry.options["room_max_age_minutes"] == 45 + assert hass.data[DOMAIN][entry.entry_id].config["room_max_age_minutes"] == 45 + + +async def test_v1_entry_migrates_on_setup(hass, inputs): + entry = MockConfigEntry(domain=DOMAIN, version=1, data={ + "climate_entity": "climate.reference", "heater_entity": "switch.boiler", + "room_entities": ["sensor.room"], "outside_entity": "sensor.outside"}) + entry.add_to_hass(hass) + assert await hass.config_entries.async_setup(entry.entry_id) + await hass.async_block_till_done() + assert entry.version == 2 + assert entry.data["outside_temperature_sources"] == ["sensor.outside"] + assert "outside_entity" not in entry.data diff --git a/tests/ha/test_lifecycle.py b/tests/ha/test_lifecycle.py new file mode 100644 index 0000000..27d2034 --- /dev/null +++ b/tests/ha/test_lifecycle.py @@ -0,0 +1,148 @@ +"""Run the observer through real HA lifecycle and storage APIs.""" +from datetime import timedelta +from unittest.mock import patch + +import pytest +from homeassistant.util import dt as dt_util +from pytest_homeassistant_custom_component.common import async_fire_time_changed + +from homeassistant.helpers import entity_registry as er +from homeassistant.helpers.storage import Store + +from custom_components.heating_mpc.const import DOMAIN + + +async def setup(hass, entry): + entry.add_to_hass(hass) + assert await hass.config_entries.async_setup(entry.entry_id) + await hass.async_block_till_done() + return hass.data[DOMAIN][entry.entry_id] + + +async def test_setup_reload_unload(hass, entry, inputs): + coordinator = await setup(hass, entry) + registry = er.async_get(hass) + entities = er.async_entries_for_config_entry(registry, entry.entry_id) + assert len(entities) == 5 + ids = {e.unique_id: e.entity_id for e in entities} + assert coordinator.data["status"] == "learning" + proposal_id = registry.async_get_entity_id("binary_sensor", DOMAIN, f"{entry.entry_id}_proposal") + assert hass.states.get(proposal_id).state == "unavailable" + assert await hass.config_entries.async_reload(entry.entry_id) + await hass.async_block_till_done() + assert hass.data[DOMAIN][entry.entry_id] is not coordinator + assert ids == {e.unique_id: e.entity_id for e in er.async_entries_for_config_entry(registry, entry.entry_id)} + assert await hass.config_entries.async_unload(entry.entry_id) + await hass.async_block_till_done() + assert entry.entry_id not in hass.data[DOMAIN] + assert not coordinator._listeners + assert coordinator._unsub_refresh is None + + +async def test_options_reload_removes_only_old_room(hass, entry, inputs): + await setup(hass, entry) + registry = er.async_get(hass) + old = registry.async_get_entity_id("sensor", DOMAIN, f"{entry.entry_id}_sensor.room_predicted_60min") + hass.states.async_set("sensor.new", "21", {"unit_of_measurement": "°C"}) + hass.config_entries.async_update_entry(entry, options={"room_entities": ["sensor.new"]}) + await hass.async_block_till_done() + assert hass.data[DOMAIN][entry.entry_id].rooms == ["sensor.new"] + assert registry.async_get(old) is None + assert registry.async_get_entity_id("sensor", DOMAIN, f"{entry.entry_id}_sensor.new_predicted_60min") + assert len(er.async_entries_for_config_entry(registry, entry.entry_id)) == 5 + + +async def test_storage_restore_and_remove(hass, entry, inputs): + coordinator = await setup(hass, entry) + model = coordinator.learners["sensor.room"] + model.count, model.on_count, model.off_count = 250, 125, 125 + await coordinator._store.async_save({"schema_version": 2, "heater_entity": "switch.boiler", + "learners": {"sensor.room": model.as_dict()}}) + assert await hass.config_entries.async_reload(entry.entry_id) + await hass.async_block_till_done() + restored = hass.data[DOMAIN][entry.entry_id] + assert restored.learners["sensor.room"].count == 250 + assert restored.data["status"] == "ready" + assert await hass.config_entries.async_remove(entry.entry_id) + await hass.async_block_till_done() + assert await Store(hass, 1, f"{DOMAIN}.{entry.entry_id}").async_load() is None + assert not er.async_entries_for_config_entry(er.async_get(hass), entry.entry_id) + + +async def test_missing_sensor_manual_off_and_target(hass, entry, inputs): + coordinator = await setup(hass, entry) + hass.states.async_set("sensor.room", "unavailable") + await coordinator.async_refresh() + assert coordinator.data["status"] == "waiting_for_sensors" + hass.states.async_set("sensor.room", "20", {"unit_of_measurement": "°C"}) + hass.states.async_set("climate.reference", "off", {"temperature": 19, "preset_mode": "away"}) + await coordinator.async_refresh() + assert coordinator.data["status"] == "paused" + assert coordinator.data["proposal"] is None + assert hass.states.get("climate.reference").attributes["preset_mode"] == "away" + hass.states.async_set("climate.reference", "heat", {"temperature": 21, "preset_mode": "sleep"}) + await coordinator.async_refresh() + assert coordinator.data["target"] == 21 + assert hass.states.get("switch.boiler").state == "off" + + +async def test_optimizer_failure_is_unavailable(hass, entry, inputs): + coordinator = await setup(hass, entry) + model = coordinator.learners["sensor.room"] + model.count, model.on_count, model.off_count = 250, 125, 125 + coordinator._update_counts() + with patch("custom_components.heating_mpc.coordinator.optimize", side_effect=RuntimeError("test")): + await coordinator.async_refresh() + assert coordinator.data["status"] == "prediction_error" + assert coordinator.data["proposal"] is None + + +async def test_scheduled_sample_saves_and_unload_stops_polling(hass, entry, inputs, freezer, hass_storage): + coordinator = await setup(hass, entry) + future = dt_util.utcnow() + timedelta(minutes=5, seconds=1) + freezer.move_to(future) + async_fire_time_changed(hass, future) + await hass.async_block_till_done() + assert coordinator.learners["sensor.room"].count == 1 + payload = hass_storage[f"{DOMAIN}.{entry.entry_id}"]["data"] + assert payload["learners"]["sensor.room"]["off_count"] == 1 + assert await hass.config_entries.async_unload(entry.entry_id) + future += timedelta(minutes=5) + freezer.move_to(future) + async_fire_time_changed(hass, future) + await hass.async_block_till_done() + assert coordinator.learners["sensor.room"].count == 1 + + +@pytest.mark.parametrize("payload", ["broken", {"learners": []}, {"learners": {"sensor.room": {"theta": []}}}]) +async def test_corrupt_saved_payload_starts_learning(hass, entry, inputs, hass_storage, payload): + hass_storage[f"{DOMAIN}.{entry.entry_id}"] = {"version": 1, "data": payload} + coordinator = await setup(hass, entry) + assert coordinator.data["status"] == "learning" + assert coordinator.data["samples"] == 0 + + +async def test_startup_preserves_manual_off_with_trained_model(hass, entry, inputs, hass_storage): + from custom_components.heating_mpc.physics import Learner + model = Learner(count=250, on_count=125, off_count=125) + hass_storage[f"{DOMAIN}.{entry.entry_id}"] = {"version": 1, "data": { + "schema_version": 2, "heater_entity": "switch.boiler", + "learners": {"sensor.room": model.as_dict()}}} + hass.states.async_set("climate.reference", "off", {"temperature": 19, "preset_mode": "away"}) + coordinator = await setup(hass, entry) + assert coordinator.data["status"] == "paused" + assert coordinator.data["proposal"] is None + assert hass.states.get("climate.reference").state == "off" + assert hass.states.get("climate.reference").attributes["temperature"] == 19 + + +async def test_startup_missing_inputs_then_recovery(hass, entry): + coordinator = await setup(hass, entry) + assert coordinator.data["status"] == "waiting_for_sensors" + hass.states.async_set("sensor.room", "20", {"unit_of_measurement": "°C"}) + hass.states.async_set("sensor.outside", "5", {"unit_of_measurement": "°C"}) + hass.states.async_set("switch.boiler", "off") + hass.states.async_set("climate.reference", "heat", {"temperature": 22}) + await coordinator.async_refresh() + assert coordinator.data["status"] == "learning" + assert coordinator.data["samples"] == 0