Stop publishing and saving from the start of unload; keep deletions final

Home Assistant marks a coordinator shut down only after the platforms are
unloaded, and the model save checked for shutdown before waiting for the
storage lock. A deletion could therefore be followed by a save that had
been waiting for it, recreating the file. Also, unloading cancels a running
scheduled update, which released the lock while the file write continued
in a worker thread.

- The coordinator is marked closing before platforms unload (restored if
  unloading fails); closing updates, relay callbacks and saves do nothing.
- The save checks closing and a removed-model mark after acquiring the
  lock, and runs shielded so the lock is held until the write finishes.
- Removal marks the model removed under the lock before deleting it.
- A failed setup also shuts the coordinator down.

Tests: unload and reload during a running optimization, removal during a
save, a save waiting for a deletion, simultaneous saves, repeated
setup/unload, shutdown ordering and failed setup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/ha/test_concurrency.py

@@ -0,0 +1,134 @@
+"""Unload, reload and removal while work is still in flight, in the real HA lifecycle."""
+import asyncio
+import threading
+from datetime import timedelta
+from unittest.mock import patch
+
+from homeassistant.helpers import entity_registry as er
+from homeassistant.helpers.storage import Store
+from homeassistant.util import dt as dt_util
+from pytest_homeassistant_custom_component.common import async_fire_time_changed
+
+from custom_components.heating_mpc import coordinator as coordinator_module
+from custom_components.heating_mpc.const import DOMAIN
+
+REAL_OPTIMIZE = coordinator_module.optimize
+
+
+async def setup(hass, entry):
+    entry.add_to_hass(hass)
+    assert await hass.config_entries.async_setup(entry.entry_id)
+    await hass.async_block_till_done()
+    return hass.data[DOMAIN][entry.entry_id]
+
+
+def trained(coordinator):
+    model = coordinator.learners["sensor.room"]
+    model.count, model.on_count, model.off_count = 250, 125, 125
+    coordinator._update_counts()
+
+
+class BlockedOptimizer:
+    """Holds the optimizer in its worker thread until released."""
+
+    def __init__(self):
+        self.started, self.release = threading.Event(), threading.Event()
+
+    def __call__(self, *args, **kwargs):
+        self.started.set()
+        self.release.wait(5)
+        return REAL_OPTIMIZE(*args, **kwargs)
+
+    async def wait_started(self):
+        # Polled on the event loop: the test executor may run one job at a time.
+        for _ in range(500):
+            if self.started.is_set():
+                return True
+            await asyncio.sleep(0.01)
+        return False
+
+
+def status_id(hass, entry):
+    return er.async_get(hass).async_get_entity_id("sensor", DOMAIN, f"{entry.entry_id}_status")
+
+
+async def test_unload_during_optimization_publishes_nothing(hass, entry, inputs):
+    coordinator = await setup(hass, entry)
+    trained(coordinator)
+    shown = coordinator.data
+    blocked = BlockedOptimizer()
+    with patch.object(coordinator_module, "optimize", blocked):
+        refresh = hass.async_create_task(coordinator.async_refresh())
+        assert await blocked.wait_started()
+        assert await hass.config_entries.async_unload(entry.entry_id)
+        state_after_unload = hass.states.get(status_id(hass, entry))
+        blocked.release.set()
+        await refresh
+        await hass.async_block_till_done()
+    # The plan finished after unloading began: kept out of the data and the states.
+    assert coordinator.data is shown
+    assert coordinator.data["status"] == "learning"
+    assert hass.states.get(status_id(hass, entry)) == state_after_unload
+
+
+async def test_reload_during_optimization_shows_only_the_new_coordinator(hass, entry, inputs):
+    old = await setup(hass, entry)
+    trained(old)  # only in memory: the reloaded coordinator is not ready
+    blocked = BlockedOptimizer()
+    with patch.object(coordinator_module, "optimize", blocked):
+        refresh = hass.async_create_task(old.async_refresh())
+        assert await blocked.wait_started()
+        assert await hass.config_entries.async_reload(entry.entry_id)
+        await hass.async_block_till_done()
+        new = hass.data[DOMAIN][entry.entry_id]
+        blocked.release.set()
+        await refresh
+        await hass.async_block_till_done()
+    assert new is not old and old.closing
+    assert old.data["status"] != "ready"
+    assert hass.states.get(status_id(hass, entry)).state == new.data["status"] == "learning"
+
+
+async def test_removal_during_a_save_leaves_no_model_behind(hass, entry, inputs, freezer):
+    coordinator = await setup(hass, entry)
+    started, release = asyncio.Event(), asyncio.Event()
+    write = coordinator._store.async_save
+
+    async def slow_save(data):
+        started.set()
+        await release.wait()
+        await write(data)
+    coordinator._store.async_save = slow_save
+    future = dt_util.utcnow() + timedelta(minutes=5, seconds=1)
+    freezer.move_to(future)
+    async_fire_time_changed(hass, future)  # a learning poll, which saves
+    await asyncio.wait_for(started.wait(), 5)
+    removal = hass.async_create_task(hass.config_entries.async_remove(entry.entry_id))
+    for _ in range(5):
+        await asyncio.sleep(0)
+    assert not removal.done()  # deletion waits for the save in progress
+    release.set()
+    assert await removal
+    await hass.async_block_till_done()
+    assert await Store(hass, 1, f"{DOMAIN}.{entry.entry_id}").async_load() is None
+
+
+async def test_repeated_setup_and_unload_leave_nothing_running(hass, entry, inputs, freezer):
+    entry.add_to_hass(hass)
+    coordinators = []
+    for _ in range(3):
+        assert await hass.config_entries.async_setup(entry.entry_id)
+        await hass.async_block_till_done()
+        coordinators.append(hass.data[DOMAIN][entry.entry_id])
+        assert await hass.config_entries.async_unload(entry.entry_id)
+        await hass.async_block_till_done()
+    assert entry.entry_id not in hass.data[DOMAIN]
+    assert all(c.closing and c._unsub_refresh is None and not c._listeners for c in coordinators)
+    counts = [c.learners["sensor.room"].count for c in coordinators]
+    hass.states.async_set("switch.boiler", "on")
+    future = dt_util.utcnow() + timedelta(minutes=10)
+    freezer.move_to(future)
+    async_fire_time_changed(hass, future)
+    await hass.async_block_till_done()
+    assert not any(c._relay_events for c in coordinators)
+    assert [c.learners["sensor.room"].count for c in coordinators] == counts

tests/test_coordinator.py

@@ -24,6 +24,9 @@ class CoordinatorBase:
     async def async_config_entry_first_refresh(self):
         self.data = await self._async_update_data()
 
+    async def async_shutdown(self):
+        self._shutdown_requested = True
+
 
 class Store:
     def __init__(self, *args):
@@ -72,6 +75,7 @@ class BoundaryTests(unittest.IsolatedAsyncioTestCase):
             config=types.SimpleNamespace(units=types.SimpleNamespace(temperature_unit='°C')),
             services=types.SimpleNamespace(async_call=Mock(side_effect=AssertionError('No service calls permitted'))),
             track_state=Mock(return_value=Mock(name='unsub_relay')),
+            async_create_task=lambda coro, name=None, eager_start=True: asyncio.ensure_future(coro),
             registry=types.SimpleNamespace(entries=[], async_remove=Mock()),
             config_entries=types.SimpleNamespace(
                 async_forward_entry_setups=AsyncMock(), async_unload_platforms=AsyncMock(return_value=True),
@@ -1308,6 +1312,82 @@ class BoundaryTests(unittest.IsolatedAsyncioTestCase):
         self.assertIs(await self.sample(), previous)
         worker.assert_not_called()
 
+    async def test_unload_marks_shutdown_before_platforms_unload(self):
+        await integration.async_setup_entry(self.hass, self.entry)
+        c = self.hass.data['heating_mpc']['entry']
+        seen = []
+        async def unload_platforms(entry, platforms):
+            seen.append(c.closing)
+            return True
+        self.hass.config_entries.async_unload_platforms.side_effect = unload_platforms
+        self.assertTrue(await integration.async_unload_entry(self.hass, self.entry))
+        self.assertEqual(seen, [True])
+        # A failed unload leaves the entry loaded, so the coordinator carries on.
+        await integration.async_setup_entry(self.hass, self.entry)
+        c = self.hass.data['heating_mpc']['entry']
+        self.hass.config_entries.async_unload_platforms.side_effect = None
+        self.hass.config_entries.async_unload_platforms.return_value = False
+        self.assertFalse(await integration.async_unload_entry(self.hass, self.entry))
+        self.assertFalse(c.closing)
+
+    async def test_nothing_is_learned_saved_or_recorded_once_unloading_began(self):
+        self.ready_office()
+        await self.sample()
+        shown = self.c.data = {'status': 'learning', 'proposal': None}
+        self.fresh('sensor.room', 'sensor.outside', 'sensor.ups_power')
+        self.c.async_begin_shutdown()
+        count = self.c.learners['office'].count
+        self.assertIs(await self.sample(), shown)
+        self.assertEqual(self.c.learners['office'].count, count)
+        self.c._store.async_save.assert_not_awaited()
+        self.assertFalse(await self.c._async_save('sensor.outside'))
+        # A relay callback queued before the listener was removed is ignored.
+        new = types.SimpleNamespace(state='on', last_changed=self.now)
+        self.c.async_relay_changed(types.SimpleNamespace(
+            data={'old_state': types.SimpleNamespace(state='off'), 'new_state': new}, time_fired=self.now))
+        self.assertFalse(self.c._relay_events)
+
+    async def test_failed_setup_shuts_the_coordinator_down(self):
+        self.hass.config_entries.async_forward_entry_setups.side_effect = RuntimeError('test')
+        with patch.object(coordinator.HeatingMPCCoordinator, 'async_shutdown', autospec=True) as shutdown:
+            with self.assertRaises(RuntimeError):
+                await integration.async_setup_entry(self.hass, self.entry)
+        shutdown.assert_awaited_once()
+
+    async def test_simultaneous_saves_never_overlap(self):
+        self.ready_office()
+        order = []
+        async def slow_save(payload):
+            order.append('start')
+            await asyncio.sleep(0)
+            await asyncio.sleep(0)
+            order.append('end')
+        self.c._store.async_save.side_effect = slow_save
+        results = await asyncio.gather(*(self.c._async_save('sensor.outside') for _ in range(3)))
+        self.assertEqual(results, [True, True, True])
+        self.assertEqual(order, ['start', 'end'] * 3)
+
+    async def test_a_save_waiting_for_a_deletion_does_not_recreate_the_model(self):
+        self.ready_office()
+        order, release = [], asyncio.Event()
+        store = Store()
+        async def slow_remove():
+            order.append('delete started')
+            await release.wait()
+            order.append('deleted')
+        store.async_remove.side_effect = slow_remove
+        with patch.object(integration, 'Store', return_value=store):
+            removal = asyncio.ensure_future(integration.async_remove_entry(self.hass, self.entry))
+            await asyncio.sleep(0)
+            saving = asyncio.ensure_future(self.c._async_save('sensor.outside'))
+            await asyncio.sleep(0)
+            self.assertEqual(order, ['delete started'])  # the save is waiting for the lock
+            release.set()
+            await removal
+            self.assertFalse(await saving)
+        self.assertEqual(order, ['delete started', 'deleted'])
+        self.c._store.async_save.assert_not_awaited()
+
     async def test_restore_waits_for_a_save_still_in_progress(self):
         self.ready_office()
         order = []