From 682b8e05355f14f43a87fafd9d3f576acea333c3 Mon Sep 17 00:00:00 2001
From: Mike Degatano <michael.degatano@gmail.com>
Date: Wed, 4 Oct 2023 12:54:42 -0400
Subject: [PATCH] Core API check during startup can timeout (#4595)

* Core API check during startup can timeout

* Use a more specific exception so caller can differentiate
---
 supervisor/exceptions.py         |  4 ++++
 supervisor/homeassistant/core.py | 23 ++++++++++++++------
 tests/homeassistant/test_core.py | 36 ++++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 6 deletions(-)

diff --git a/supervisor/exceptions.py b/supervisor/exceptions.py
index 0b44f63bfc6..a24242c0bc7 100644
--- a/supervisor/exceptions.py
+++ b/supervisor/exceptions.py
@@ -67,6 +67,10 @@ class HomeAssistantCrashError(HomeAssistantError):
     """Error on crash of a Home Assistant startup."""
 
 
+class HomeAssistantStartupTimeout(HomeAssistantCrashError):
+    """Timeout waiting for Home Assistant successful startup."""
+
+
 class HomeAssistantAPIError(HomeAssistantError):
     """Home Assistant API exception."""
 
diff --git a/supervisor/homeassistant/core.py b/supervisor/homeassistant/core.py
index d337bb5dbe2..6a5374e5f18 100644
--- a/supervisor/homeassistant/core.py
+++ b/supervisor/homeassistant/core.py
@@ -2,12 +2,14 @@
 import asyncio
 from collections.abc import Awaitable
 from contextlib import suppress
+from dataclasses import dataclass
+from datetime import datetime, timedelta
 import logging
 import re
 import secrets
 import shutil
+from typing import Final
 
-import attr
 from awesomeversion import AwesomeVersion
 
 from ..const import ATTR_HOMEASSISTANT, BusEvent
@@ -21,6 +23,7 @@
     HomeAssistantCrashError,
     HomeAssistantError,
     HomeAssistantJobError,
+    HomeAssistantStartupTimeout,
     HomeAssistantUpdateError,
     JobException,
 )
@@ -40,15 +43,17 @@
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
 
+SECONDS_BETWEEN_API_CHECKS: Final[int] = 5
+STARTUP_API_CHECK_TIMEOUT: Final[timedelta] = timedelta(minutes=5)
 RE_YAML_ERROR = re.compile(r"homeassistant\.util\.yaml")
 
 
-@attr.s(frozen=True)
+@dataclass
 class ConfigResult:
     """Return object from config check."""
 
-    valid = attr.ib()
-    log = attr.ib()
+    valid: bool
+    log: str
 
 
 class HomeAssistantCore(JobGroup):
@@ -435,8 +440,9 @@ async def _block_till_run(self, version: AwesomeVersion) -> None:
             return
         _LOGGER.info("Wait until Home Assistant is ready")
 
-        while True:
-            await asyncio.sleep(5)
+        start = datetime.now()
+        while not (timeout := datetime.now() >= start + STARTUP_API_CHECK_TIMEOUT):
+            await asyncio.sleep(SECONDS_BETWEEN_API_CHECKS)
 
             # 1: Check if Container is is_running
             if not await self.instance.is_running():
@@ -450,6 +456,11 @@ async def _block_till_run(self, version: AwesomeVersion) -> None:
                 return
 
         self._error_state = True
+        if timeout:
+            raise HomeAssistantStartupTimeout(
+                "No API response in 5 minutes, assuming core has had a fatal startup error",
+                _LOGGER.error,
+            )
         raise HomeAssistantCrashError()
 
     @Job(
diff --git a/tests/homeassistant/test_core.py b/tests/homeassistant/test_core.py
index 671fc758eff..e2f89bfdc79 100644
--- a/tests/homeassistant/test_core.py
+++ b/tests/homeassistant/test_core.py
@@ -1,9 +1,12 @@
 """Test Home Assistant core."""
+
+from datetime import datetime, timedelta
 from unittest.mock import MagicMock, Mock, PropertyMock, patch
 
 from awesomeversion import AwesomeVersion
 from docker.errors import DockerException, ImageNotFound, NotFound
 import pytest
+from time_machine import travel
 
 from supervisor.const import CpuArch
 from supervisor.coresys import CoreSys
@@ -14,6 +17,7 @@
     AudioUpdateError,
     CodeNotaryError,
     DockerError,
+    HomeAssistantCrashError,
     HomeAssistantError,
     HomeAssistantJobError,
 )
@@ -263,3 +267,35 @@ async def test_stats_failures(
 
     with pytest.raises(HomeAssistantError):
         await coresys.homeassistant.core.stats()
+
+
+async def test_api_check_timeout(
+    coresys: CoreSys, container: MagicMock, caplog: pytest.LogCaptureFixture
+):
+    """Test attempts to contact the API timeout."""
+    container.status = "stopped"
+    coresys.homeassistant.version = AwesomeVersion("2023.9.0")
+    coresys.homeassistant.api.check_api_state.return_value = False
+
+    async def mock_instance_start(*_):
+        container.status = "running"
+
+    with patch.object(
+        DockerHomeAssistant, "start", new=mock_instance_start
+    ), patch.object(DockerAPI, "container_is_initialized", return_value=True), travel(
+        datetime(2023, 10, 2, 0, 0, 0), tick=False
+    ) as traveller:
+
+        async def mock_sleep(*args):
+            traveller.shift(timedelta(minutes=1))
+
+        with patch(
+            "supervisor.homeassistant.core.asyncio.sleep", new=mock_sleep
+        ), pytest.raises(HomeAssistantCrashError):
+            await coresys.homeassistant.core.start()
+
+    assert coresys.homeassistant.api.check_api_state.call_count == 5
+    assert (
+        "No API response in 5 minutes, assuming core has had a fatal startup error"
+        in caplog.text
+    )