Skip to content

Commit

Permalink
Add storage helper and migrate config entries (home-assistant#15045)
Browse files Browse the repository at this point in the history
* Add storage helper

* Migrate config entries to use the storage helper

* Make sure tests do not do I/O

* Lint

* Add versions to stored data

* Add more instance variables

* Make migrator load config if nothing to migrate

* Address comments
  • Loading branch information
balloob authored Jun 25, 2018
1 parent 672a3c7 commit ae51dc0
Show file tree
Hide file tree
Showing 8 changed files with 384 additions and 46 deletions.
2 changes: 1 addition & 1 deletion homeassistant/components/sensor/fitbit.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ def setup_platform(hass, config, add_devices, discovery_info=None):
hass, config, add_devices, config_path, discovery_info=None)
return False
else:
config_file = save_json(config_path, DEFAULT_CONFIG)
save_json(config_path, DEFAULT_CONFIG)
request_app_setup(
hass, config, add_devices, config_path, discovery_info=None)
return False
Expand Down
60 changes: 28 additions & 32 deletions homeassistant/config_entries.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,15 +112,13 @@ async def async_step_discovery(info):
"""

import logging
import os
import uuid

from . import data_entry_flow
from .core import callback
from .exceptions import HomeAssistantError
from .setup import async_setup_component, async_process_deps_reqs
from .util.json import load_json, save_json
from .util.decorator import Registry
from homeassistant import data_entry_flow
from homeassistant.core import callback
from homeassistant.exceptions import HomeAssistantError
from homeassistant.setup import async_setup_component, async_process_deps_reqs
from homeassistant.util.decorator import Registry


_LOGGER = logging.getLogger(__name__)
Expand All @@ -136,6 +134,10 @@ async def async_step_discovery(info):
]


STORAGE_KEY = 'core.config_entries'
STORAGE_VERSION = 1

# Deprecated since 0.73
PATH_CONFIG = '.config_entries.json'

SAVE_DELAY = 1
Expand Down Expand Up @@ -271,7 +273,7 @@ def __init__(self, hass, hass_config):
hass, self._async_create_flow, self._async_finish_flow)
self._hass_config = hass_config
self._entries = None
self._sched_save = None
self._store = hass.helpers.storage.Store(STORAGE_VERSION, STORAGE_KEY)

@callback
def async_domains(self):
Expand Down Expand Up @@ -305,7 +307,7 @@ async def async_remove(self, entry_id):
raise UnknownEntry

entry = self._entries.pop(found)
self._async_schedule_save()
await self._async_schedule_save()

unloaded = await entry.async_unload(self.hass)

Expand All @@ -314,14 +316,14 @@ async def async_remove(self, entry_id):
}

async def async_load(self):
"""Load the config."""
path = self.hass.config.path(PATH_CONFIG)
if not os.path.isfile(path):
self._entries = []
return
"""Handle loading the config."""
# Migrating for config entries stored before 0.73
config = await self.hass.helpers.storage.async_migrator(
self.hass.config.path(PATH_CONFIG), self._store,
old_conf_migrate_func=_old_conf_migrator
)

entries = await self.hass.async_add_job(load_json, path)
self._entries = [ConfigEntry(**entry) for entry in entries]
self._entries = [ConfigEntry(**entry) for entry in config['entries']]

async def async_forward_entry_setup(self, entry, component):
"""Forward the setup of an entry to a different component.
Expand Down Expand Up @@ -372,7 +374,7 @@ async def _async_finish_flow(self, result):
source=result['source'],
)
self._entries.append(entry)
self._async_schedule_save()
await self._async_schedule_save()

# Setup entry
if entry.domain in self.hass.config.components:
Expand Down Expand Up @@ -416,20 +418,14 @@ async def _async_create_flow(self, handler, *, source, data):

return handler()

@callback
def _async_schedule_save(self):
"""Schedule saving the entity registry."""
if self._sched_save is not None:
self._sched_save.cancel()

self._sched_save = self.hass.loop.call_later(
SAVE_DELAY, self.hass.async_add_job, self._async_save
)

async def _async_save(self):
async def _async_schedule_save(self):
"""Save the entity registry to a file."""
self._sched_save = None
data = [entry.as_dict() for entry in self._entries]
data = {
'entries': [entry.as_dict() for entry in self._entries]
}
await self._store.async_save(data, delay=SAVE_DELAY)


await self.hass.async_add_job(
save_json, self.hass.config.path(PATH_CONFIG), data)
async def _old_conf_migrator(old_config):
"""Migrate the pre-0.73 config format to the latest version."""
return {'entries': old_config}
14 changes: 14 additions & 0 deletions homeassistant/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,20 @@ def async_add_job(

return task

@callback
def async_add_executor_job(
self,
target: Callable[..., Any],
*args: Any) -> asyncio.tasks.Task:
"""Add an executor job from within the event loop."""
task = self.loop.run_in_executor(None, target, *args)

# If a task is scheduled
if self._track_task:
self._pending_tasks.append(task)

return task

@callback
def async_track_tasks(self):
"""Track tasks so you can wait for all tasks to be done."""
Expand Down
157 changes: 157 additions & 0 deletions homeassistant/helpers/storage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
"""Helper to help store data."""
import asyncio
import logging
import os
from typing import Dict, Optional

from homeassistant.const import EVENT_HOMEASSISTANT_STOP
from homeassistant.core import callback
from homeassistant.loader import bind_hass
from homeassistant.util import json
from homeassistant.helpers.event import async_call_later

STORAGE_DIR = '.storage'
_LOGGER = logging.getLogger(__name__)


@bind_hass
async def async_migrator(hass, old_path, store, *, old_conf_migrate_func=None):
"""Helper function to migrate old data to a store and then load data.
async def old_conf_migrate_func(old_data)
"""
def load_old_config():
"""Helper to load old config."""
if not os.path.isfile(old_path):
return None

return json.load_json(old_path)

config = await hass.async_add_executor_job(load_old_config)

if config is None:
return await store.async_load()

if old_conf_migrate_func is not None:
config = await old_conf_migrate_func(config)

await store.async_save(config)
await hass.async_add_executor_job(os.remove, old_path)
return config


@bind_hass
class Store:
"""Class to help storing data."""

def __init__(self, hass, version: int, key: str):
"""Initialize storage class."""
self.version = version
self.key = key
self.hass = hass
self._data = None
self._unsub_delay_listener = None
self._unsub_stop_listener = None
self._write_lock = asyncio.Lock()

@property
def path(self):
"""Return the config path."""
return self.hass.config.path(STORAGE_DIR, self.key)

async def async_load(self):
"""Load data.
If the expected version does not match the given version, the migrate
function will be invoked with await migrate_func(version, config).
"""
if self._data is not None:
data = self._data
else:
data = await self.hass.async_add_executor_job(
json.load_json, self.path, None)

if data is None:
return {}

if data['version'] == self.version:
return data['data']

return await self._async_migrate_func(data['version'], data['data'])

async def async_save(self, data: Dict, *, delay: Optional[int] = None):
"""Save data with an optional delay."""
self._data = {
'version': self.version,
'key': self.key,
'data': data,
}

self._async_cleanup_delay_listener()

if delay is None:
self._async_cleanup_stop_listener()
await self._async_handle_write_data()
return

self._unsub_delay_listener = async_call_later(
self.hass, delay, self._async_callback_delayed_write)

self._async_ensure_stop_listener()

@callback
def _async_ensure_stop_listener(self):
"""Ensure that we write if we quit before delay has passed."""
if self._unsub_stop_listener is None:
self._unsub_stop_listener = self.hass.bus.async_listen_once(
EVENT_HOMEASSISTANT_STOP, self._async_callback_stop_write)

@callback
def _async_cleanup_stop_listener(self):
"""Clean up a stop listener."""
if self._unsub_stop_listener is not None:
self._unsub_stop_listener()
self._unsub_stop_listener = None

@callback
def _async_cleanup_delay_listener(self):
"""Clean up a delay listener."""
if self._unsub_delay_listener is not None:
self._unsub_delay_listener()
self._unsub_delay_listener = None

async def _async_callback_delayed_write(self, _now):
"""Handle a delayed write callback."""
self._unsub_delay_listener = None
self._async_cleanup_stop_listener()
await self._async_handle_write_data()

async def _async_callback_stop_write(self, _event):
"""Handle a write because Home Assistant is stopping."""
self._unsub_stop_listener = None
self._async_cleanup_delay_listener()
await self._async_handle_write_data()

async def _async_handle_write_data(self, *_args):
"""Handler to handle writing the config."""
data = self._data
self._data = None

async with self._write_lock:
try:
await self.hass.async_add_executor_job(
self._write_data, self.path, data)
except (json.SerializationError, json.WriteError) as err:
_LOGGER.error('Error writing config for %s: %s', self.key, err)

def _write_data(self, path: str, data: Dict):
"""Write the data."""
if not os.path.isdir(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))

_LOGGER.debug('Writing data for %s', self.key)
json.save_json(path, data)

async def _async_migrate_func(self, old_version, old_data):
"""Migrate to the new version."""
raise NotImplementedError
14 changes: 10 additions & 4 deletions homeassistant/util/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,14 @@
_UNDEFINED = object()


class SerializationError(HomeAssistantError):
"""Error serializing the data to JSON."""


class WriteError(HomeAssistantError):
"""Error writing the data."""


def load_json(filename: str, default: Union[List, Dict] = _UNDEFINED) \
-> Union[List, Dict]:
"""Load JSON data from a file and return as dict or list.
Expand Down Expand Up @@ -41,13 +49,11 @@ def save_json(filename: str, data: Union[List, Dict]):
data = json.dumps(data, sort_keys=True, indent=4)
with open(filename, 'w', encoding='utf-8') as fdesc:
fdesc.write(data)
return True
except TypeError as error:
_LOGGER.exception('Failed to serialize to JSON: %s',
filename)
raise HomeAssistantError(error)
raise SerializationError(error)
except OSError as error:
_LOGGER.exception('Saving JSON file failed: %s',
filename)
raise HomeAssistantError(error)
return False
raise WriteError(error)
8 changes: 5 additions & 3 deletions tests/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from homeassistant.setup import setup_component, async_setup_component
from homeassistant.config import async_process_component_config
from homeassistant.helpers import (
intent, entity, restore_state, entity_registry,
intent, entity, restore_state, entity_registry,
entity_platform)
from homeassistant.util.unit_system import METRIC_SYSTEM
import homeassistant.util.dt as date_util
Expand Down Expand Up @@ -110,8 +110,6 @@ def stop_hass():
def async_test_home_assistant(loop):
"""Return a Home Assistant object pointing at test config dir."""
hass = ha.HomeAssistant(loop)
hass.config_entries = config_entries.ConfigEntries(hass, {})
hass.config_entries._entries = []
hass.config.async_load = Mock()
store = auth.AuthStore(hass)
hass.auth = auth.AuthManager(hass, store, {})
Expand All @@ -137,6 +135,10 @@ def async_add_job(target, *args):
hass.config.units = METRIC_SYSTEM
hass.config.skip_pip = True

hass.config_entries = config_entries.ConfigEntries(hass, {})
hass.config_entries._entries = []
hass.config_entries._store._async_ensure_stop_listener = lambda: None

hass.state = ha.CoreState.running

# Mock async_start
Expand Down
Loading

0 comments on commit ae51dc0

Please sign in to comment.