diff --git a/src/tribler-core/tribler_core/components/simple_storage/__init__.py b/src/tribler-core/tribler_core/components/simple_storage/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/tribler-core/tribler_core/components/simple_storage/simple_storage.py b/src/tribler-core/tribler_core/components/simple_storage/simple_storage.py new file mode 100644 index 00000000000..944f1ac6a4c --- /dev/null +++ b/src/tribler-core/tribler_core/components/simple_storage/simple_storage.py @@ -0,0 +1,81 @@ +import asyncio +import logging + +from pydantic import BaseModel + +from tribler_core.utilities.path_util import Path + +DEFAULT_SAVE_INTERVAL = 5 * 60 # force Storage to save data every 5 minutes + + +class StorageData(BaseModel): + last_processed_torrent_id: int = 0 + + +class SimpleStorage: + """ SimpleStorage is object storage that stores data in JSON format and uses + `pydantic` `BaseModel` for defining models. + + It stores data on a shutdown and every 5 minutes. + + limitations: + * No transactions: last five-minute changes can be lost on Tribler crash, so the + application code should be tolerable to this and be ready, for example, to + process the same torrents again after the Tribler restart. + * If two instances of the application try to use the same storage simultaneously, + they will not see the changes made by another instance. + """ + + def __init__(self, path: Path, save_interval: float = DEFAULT_SAVE_INTERVAL): + """ + Args: + path: path to the file with storage. Could be a path to a non existent file. + save_interval: interval in seconds in which the storage will store a data to a disk. + """ + self.logger = logging.getLogger(self.__class__.__name__) + self.data = StorageData() + + self.path = path + self.save_interval = save_interval + + self._loop = asyncio.get_event_loop() + self._task: asyncio.TimerHandle = self._loop.call_later(self.save_interval, self._save_and_schedule_next) + + def load(self) -> bool: + """ Load data from `self.path`. In case the file doesn't exist, the function + will create the data with defaults values. + """ + self.logger.info(f'Loading storage from {self.path}') + loaded = False + + try: + self.data = StorageData.parse_file(self.path) + except FileNotFoundError: + self.logger.info('The storage file does not exist.') + except Exception as e: # pylint: disable=broad-except + self.logger.exception(e) + else: + loaded = True + self.logger.info(f'Loaded storage: {self.data}') + + if not loaded: + self.logger.info('Create a new storage.') + self.data = StorageData() + + return loaded + + def save(self): + """ Save data to the `self.path`. + """ + self.logger.info(f'Saving storage to: {self.path}.\nStorage {self.data}') + self.path.write_text(self.data.json()) + + def _save_and_schedule_next(self): + """ Save data and schedule the next call of save function after `self.save_interval` + """ + self.save() + self._task = self._loop.call_later(self.save_interval, self._save_and_schedule_next) + + def shutdown(self): + self._task.cancel() + self.save() diff --git a/src/tribler-core/tribler_core/components/simple_storage/simple_storage_component.py b/src/tribler-core/tribler_core/components/simple_storage/simple_storage_component.py new file mode 100644 index 00000000000..af6298950ce --- /dev/null +++ b/src/tribler-core/tribler_core/components/simple_storage/simple_storage_component.py @@ -0,0 +1,21 @@ +from tribler_core.components.base import Component +from tribler_core.components.simple_storage.simple_storage import SimpleStorage + + +class SimpleStorageComponent(Component): + """Storage is aimed to store the limited amount of data. It is not speed efficient. + """ + + storage: SimpleStorage = None + + async def run(self): + await super().run() + + path = self.session.config.state_dir / 'storage.json' + self.storage = SimpleStorage(path) + self.storage.load() + + async def shutdown(self): + await super().shutdown() + if self.storage: + self.storage.shutdown() diff --git a/src/tribler-core/tribler_core/components/simple_storage/tests/__init__.py b/src/tribler-core/tribler_core/components/simple_storage/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/tribler-core/tribler_core/components/simple_storage/tests/test_simple_storage.py b/src/tribler-core/tribler_core/components/simple_storage/tests/test_simple_storage.py new file mode 100644 index 00000000000..3979157dd4e --- /dev/null +++ b/src/tribler-core/tribler_core/components/simple_storage/tests/test_simple_storage.py @@ -0,0 +1,74 @@ +import asyncio +from json import JSONDecodeError +from unittest.mock import Mock, call, patch + +import pytest + +from tribler_core.components.simple_storage.simple_storage import DEFAULT_SAVE_INTERVAL, SimpleStorage, StorageData + +# pylint: disable=protected-access, redefined-outer-name + + +@pytest.fixture +def simple_storage(tmp_path): + return SimpleStorage(path=tmp_path / 'storage.json') + + +def test_constructor(simple_storage: SimpleStorage): + assert simple_storage.logger + assert simple_storage.data == StorageData() + assert simple_storage.path + assert simple_storage.save_interval == DEFAULT_SAVE_INTERVAL + assert simple_storage._loop + assert simple_storage._task + + +@patch('tribler_core.components.simple_storage.simple_storage.StorageData.parse_file', + Mock(side_effect=FileNotFoundError)) +def test_load_missed_file(simple_storage: SimpleStorage): + # test that in case of missed file, default values will be created + simple_storage.data = None + simple_storage.logger.info = Mock() + assert not simple_storage.load() + assert simple_storage + simple_storage.logger.info.assert_has_calls([call('The storage file does not exist.')]) + + +@patch('tribler_core.components.simple_storage.simple_storage.StorageData.parse_file', + Mock(side_effect=JSONDecodeError)) +def test_load_corrupted_file(simple_storage: SimpleStorage): + # test that in case of corrupted file, default values will be created + simple_storage.data = None + simple_storage.logger.exception = Mock() + assert not simple_storage.load() + assert simple_storage + simple_storage.logger.exception.assert_called_once() + + +def test_load(simple_storage: SimpleStorage): + # test that in case of existed file, values will be loaded from file + simple_storage.data.last_processed_torrent_id = 100 + simple_storage.save() + + simple_storage.data.last_processed_torrent_id = 1 + assert simple_storage.load() + assert simple_storage.data.last_processed_torrent_id == 100 + + +def test_shutdown(simple_storage: SimpleStorage): + # test that on shutdown values have been saved and task has been cancelled + simple_storage.data.last_processed_torrent_id = 100 + simple_storage.shutdown() + + assert simple_storage.path.exists() + assert simple_storage._task.cancelled() + + +@pytest.mark.asyncio +async def test_save_and_schedule_next(tmp_path): + # In this test we will set up save_interval as 0.1 sec, then wait for 1 sec + # and count how many times function `save` will be called. + storage = SimpleStorage(path=tmp_path / 'storage.json', save_interval=0.1) + storage.save = Mock() + await asyncio.sleep(1) + assert 8 <= storage.save.call_count <= 10 diff --git a/src/tribler-core/tribler_core/components/simple_storage/tests/test_simple_storage_component.py b/src/tribler-core/tribler_core/components/simple_storage/tests/test_simple_storage_component.py new file mode 100644 index 00000000000..f7494114b2e --- /dev/null +++ b/src/tribler-core/tribler_core/components/simple_storage/tests/test_simple_storage_component.py @@ -0,0 +1,15 @@ +import pytest + +from tribler_core.components.base import Session +from tribler_core.components.simple_storage.simple_storage_component import SimpleStorageComponent + + +# pylint: disable=protected-access + + +@pytest.mark.asyncio +async def test_simple_storage_component(tribler_config): + # Test that component could be created without errors + async with Session(tribler_config, [SimpleStorageComponent()]).start(): + comp = SimpleStorageComponent.instance() + assert comp.started_event.is_set() and not comp.failed