From 45dea3adbd348ab49f33a30c3a3c686812a7f76c Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Mon, 28 Oct 2019 15:48:15 +0900 Subject: [PATCH 01/36] Add rollback message handler to IconInnerService * Rename old rollback() to remove_precommit_state() * Add ROLLBACK message to converter_template --- iconservice/base/type_converter_templates.py | 6 +++ iconservice/icon_inner_service.py | 47 +++++++++++++++++++- iconservice/icon_service_engine.py | 23 +++++++--- iconservice/precommit_data_manager.py | 2 +- iconservice/utils/__init__.py | 2 +- 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/iconservice/base/type_converter_templates.py b/iconservice/base/type_converter_templates.py index 8233fcb65..cb14db785 100644 --- a/iconservice/base/type_converter_templates.py +++ b/iconservice/base/type_converter_templates.py @@ -41,6 +41,7 @@ class ParamType(IntEnum): WRITE_PRECOMMIT = 400 REMOVE_PRECOMMIT = 500 + ROLLBACK = 501 VALIDATE_TRANSACTION = 600 @@ -325,6 +326,11 @@ class ConstantKeys: type_convert_templates[ParamType.REMOVE_PRECOMMIT] = type_convert_templates[ParamType.WRITE_PRECOMMIT] +type_convert_templates[ParamType.ROLLBACK] = { + ConstantKeys.BLOCK_HEIGHT: ValueType.INT, + ConstantKeys.BLOCK_HASH: ValueType.BYTES +} + type_convert_templates[ParamType.VALIDATE_TRANSACTION] = { ConstantKeys.METHOD: ValueType.STRING, ConstantKeys.PARAMS: type_convert_templates[ParamType.TRANSACTION_PARAMS_DATA] diff --git a/iconservice/icon_inner_service.py b/iconservice/icon_inner_service.py index 25504759a..ad02f1ae3 100644 --- a/iconservice/icon_inner_service.py +++ b/iconservice/icon_inner_service.py @@ -331,7 +331,7 @@ def _remove_precommit_state(self, request: dict): block_height, instant_block_hash, _ = \ self._get_block_info_for_precommit_state(converted_block_params) - self._icon_service_engine.rollback(block_height, instant_block_hash) + self._icon_service_engine.remove_precommit_state(block_height, instant_block_hash) response = MakeResponse.make_response(ExceptionCode.OK) except FatalException as e: self._log_exception(e, ICON_SERVICE_LOG_TAG) @@ -347,6 +347,51 @@ def _remove_precommit_state(self, request: dict): Logger.info(f'remove_precommit_state response with {response}', ICON_INNER_LOG_TAG) return response + @message_queue_task + async def rollback(self, request: dict): + """Go back to the state of the given previous block + + :param request: + :return: + """ + + Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() start: {request}") + + self._check_icon_service_ready() + + if self._is_thread_flag_on(EnableThreadFlag.INVOKE): + loop = get_event_loop() + response = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._rollback, request) + else: + response = self._rollback(request) + + Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() end: {response}") + + def _rollback(self, request: dict) -> dict: + Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"_rollback() start: {request}") + + response = {} + try: + converted_params = TypeConverter.convert(request, ParamType.ROLLBACK) + block_height: int = converted_params[ConstantKeys.BLOCK_HEIGHT] + block_hash: bytes = converted_params[ConstantKeys.BLOCK_HASH] + + response: dict = self._icon_service_engine.rollback(block_height, block_hash) + response = MakeResponse.make_response(response) + except FatalException as e: + self._log_exception(e, ICON_SERVICE_LOG_TAG) + response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) + self._close() + except IconServiceBaseException as icon_e: + self._log_exception(icon_e, ICON_SERVICE_LOG_TAG) + response = MakeResponse.make_error_response(icon_e.code, icon_e.message) + except BaseException as e: + self._log_exception(e, ICON_SERVICE_LOG_TAG) + response = MakeResponse.make_error_response(ExceptionCode.SYSTEM_ERROR, str(e)) + finally: + Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"_rollback() end: {response}") + return response + @message_queue_task async def validate_transaction(self, request: dict): Logger.debug(f'pre_validate_check request with {request}', ICON_INNER_LOG_TAG) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index a77d0c35a..25df620a1 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -62,10 +62,11 @@ from .precommit_data_manager import PrecommitData, PrecommitDataManager, PrecommitFlag from .prep import PRepEngine, PRepStorage from .prep.data import PRep -from .utils import print_log_with_level +from .utils import print_log_with_level, bytes_to_hex from .utils import sha3_256, int_to_bytes, ContextEngine, ContextStorage from .utils import to_camel_case, bytes_to_hex from .utils.bloom import BloomFilter +from .base.type_converter_templates import ConstantKeys if TYPE_CHECKING: from .iconscore.icon_score_event_log import EventLog @@ -1960,18 +1961,30 @@ def _process_ipc(context: 'IconScoreContext', context.engine.iiss.send_calculate(iiss_db_path, standby_db_info.block_height) wal_writer.write_state(WALState.SEND_CALCULATE.value, add=True) - def rollback(self, block_height: int, instant_block_hash: bytes) -> None: + def remove_precommit_state(self, block_height: int, instant_block_hash: bytes) -> None: """Throw away a precommit state in context.block_batch and IconScoreEngine :param block_height: height of block which is needed to be removed from the pre-commit data manager :param instant_block_hash: hash of block which is needed to be removed from the pre-commit data manager """ - Logger.warning(tag=self.TAG, msg=f"rollback() start: height={block_height}") + Logger.warning(tag=self.TAG, msg=f"remove_precommit_state() start: height={block_height}") self._precommit_data_manager.validate_precommit_block(instant_block_hash) - self._precommit_data_manager.rollback(instant_block_hash) + self._precommit_data_manager.remove_precommit_state(instant_block_hash) - Logger.warning(tag=self.TAG, msg="rollback() end") + Logger.warning(tag=self.TAG, msg="remove_precommit_state() end") + + def rollback(self, block_height: int, block_hash: bytes) -> dict: + Logger.warning(tag=self.TAG, msg=f"rollback() start: height={block_height}, hash={bytes_to_hex(block_hash)}") + + response = { + ConstantKeys.BLOCK_HEIGHT: block_height, + ConstantKeys.BLOCK_HASH: block_hash + } + + Logger.warning(tag=self.TAG, msg=f"rollback() end: height={block_height}, hash={bytes_to_hex(block_hash)}") + + return response def clear_context_stack(self): """Clear IconScoreContext stacks diff --git a/iconservice/precommit_data_manager.py b/iconservice/precommit_data_manager.py index e3807eb6c..183328b2d 100644 --- a/iconservice/precommit_data_manager.py +++ b/iconservice/precommit_data_manager.py @@ -214,7 +214,7 @@ def commit(self, block: 'Block'): # Clear remaining precommit data which have the same block height self._precommit_data_mapper.clear() - def rollback(self, instant_block_hash: bytes): + def remove_precommit_state(self, instant_block_hash: bytes): if instant_block_hash in self._precommit_data_mapper: del self._precommit_data_mapper[instant_block_hash] diff --git a/iconservice/utils/__init__.py b/iconservice/utils/__init__.py index ef26b0277..26ebf6e13 100644 --- a/iconservice/utils/__init__.py +++ b/iconservice/utils/__init__.py @@ -46,7 +46,7 @@ def byte_length_of_int(n: int): def bytes_to_hex(data: Optional[bytes], prefix: str = "0x") -> str: - if data is None: + if not isinstance(data, bytes): return "None" return f"{prefix}{data.hex()}" From 345aa9714371057c4191e71c2594ec748bf04e76 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 29 Oct 2019 15:35:20 +0900 Subject: [PATCH 02/36] Apply backup_manager to IconServiceEngine * Add BackupManager and RollbackManager to database package --- iconservice/database/backup_manager.py | 163 +++++++++++++++++++++++ iconservice/database/rollback_manager.py | 159 ++++++++++++++++++++++ iconservice/icon_constant.py | 2 + iconservice/icon_service_engine.py | 13 +- iconservice/iiss/reward_calc/msg_data.py | 7 +- 5 files changed, 341 insertions(+), 3 deletions(-) create mode 100644 iconservice/database/backup_manager.py create mode 100644 iconservice/database/rollback_manager.py diff --git a/iconservice/database/backup_manager.py b/iconservice/database/backup_manager.py new file mode 100644 index 000000000..5ee3d60ab --- /dev/null +++ b/iconservice/database/backup_manager.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from enum import Flag +from typing import TYPE_CHECKING, Optional + +from iconcommons import Logger +from .wal import WriteAheadLogWriter +from ..database.db import KeyValueDatabase +from ..icon_constant import ROLLBACK_LOG_TAG + +if TYPE_CHECKING: + from .wal import IissWAL + from ..base.block import Block + from ..database.batch import BlockBatch + from ..iconscore.icon_score_context import IconScoreContext + from ..precommit_data_manager import PrecommitData + + +TAG = ROLLBACK_LOG_TAG + + +def get_backup_filename(block_height: int) -> str: + """ + + :param block_height: the height of the block where we want to go back + :return: + """ + return f"block-{block_height}.bak" + + +class WALBackupState(Flag): + CALC_PERIOD_END_BLOCK = 1 + + +class BackupManager(object): + """Backup and rollback for the previous block state + + """ + _RC_DB = 0 + _STATE_DB = 1 + + def __init__(self, state_db_root_path: str, rc_data_path: str, icx_db: 'KeyValueDatabase'): + Logger.debug(tag=TAG, + msg=f"__init__() start: state_db_root_path={state_db_root_path}, " + f"rc_data_path={rc_data_path}") + + self._rc_data_path = rc_data_path + self._root_path = os.path.join(state_db_root_path, "backup") + self._icx_db = icx_db + + Logger.info(tag=TAG, msg=f"backup_root_path={self._root_path}") + + try: + os.mkdir(self._root_path) + except FileExistsError: + pass + + Logger.debug(tag=TAG, msg="__init__() end") + + def _get_backup_file_path(self, block_height: int) -> str: + """ + + :param block_height: the height of block to rollback to + :return: backup state file path + """ + assert block_height >= 0 + + filename = get_backup_filename(block_height) + return os.path.join(self._root_path, filename) + + def run(self, + context: 'IconScoreContext', + prev_block: 'Block', + precommit_data: 'PrecommitData', + iiss_wal: 'IissWAL', + is_calc_period_start_block: bool, + instant_block_hash: bytes): + """Backup the previous block state + + :param context: + :param prev_block: the latest confirmed block height during commit + :param precommit_data: + :param iiss_wal: + :param is_calc_period_start_block: + :param instant_block_hash: + :return: + """ + Logger.debug(tag=TAG, msg="backup() start") + + path: str = self._get_backup_file_path(prev_block.height) + Logger.info(tag=TAG, msg=f"backup_file_path={path}") + + self._clear_backup_files() + + writer = WriteAheadLogWriter( + context.revision, max_log_count=2, block=prev_block, instant_block_hash=instant_block_hash) + writer.open(path) + + if is_calc_period_start_block: + writer.write_state(WALBackupState.CALC_PERIOD_END_BLOCK.value) + + self._backup_rc_db(writer, context.storage.rc.key_value_db, iiss_wal) + self._backup_state_db(writer, self._icx_db, precommit_data.block_batch) + + writer.close() + + Logger.debug(tag=TAG, msg="backup() end") + + def _clear_backup_files(self): + try: + with os.scandir(self._root_path) as it: + for entry in it: + if entry.is_file() \ + and entry.name.startswith("block-") \ + and entry.name.endswith(".bak"): + path = os.path.join(self._root_path, entry.name) + self._remove_backup_file(path) + except BaseException as e: + Logger.info(tag=TAG, msg=str(e)) + + @classmethod + def _remove_backup_file(cls, path: str): + try: + os.remove(path) + except FileNotFoundError: + pass + except BaseException as e: + Logger.debug(tag=TAG, msg=str(e)) + + @classmethod + def _backup_rc_db(cls, writer: 'WriteAheadLogWriter', db: 'KeyValueDatabase', iiss_wal: 'IissWAL'): + def get_rc_db_generator(): + for key, _ in iiss_wal: + value: Optional[bytes] = db.get(key) + yield key, value + + writer.write_walogable(get_rc_db_generator()) + + @classmethod + def _backup_state_db(cls, writer: 'WriteAheadLogWriter', db: 'KeyValueDatabase', block_batch: 'BlockBatch'): + if block_batch is None: + block_batch = {} + + def get_state_db_generator(): + for key in block_batch: + value: Optional[bytes] = db.get(key) + yield key, value + + writer.write_walogable(get_state_db_generator()) diff --git a/iconservice/database/rollback_manager.py b/iconservice/database/rollback_manager.py new file mode 100644 index 000000000..cb20c5701 --- /dev/null +++ b/iconservice/database/rollback_manager.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +from typing import Tuple + +from iconcommons.logger import Logger +from .backup_manager import WALBackupState, get_backup_filename +from .db import KeyValueDatabase +from .wal import WriteAheadLogReader +from ..base.exception import DatabaseException +from ..icon_constant import ROLLBACK_LOG_TAG +from ..iiss.reward_calc import RewardCalcStorage +from ..iiss.reward_calc.msg_data import make_block_produce_info_key + +TAG = ROLLBACK_LOG_TAG + + +class RollbackManager(object): + def __init__(self, state_db_root_path: str, rc_data_path: str): + self._rc_data_path = rc_data_path + self._root_path = os.path.join(state_db_root_path, "backup") + + def run(self, block_height: int = -1) -> Tuple[int, bool]: + """Rollback to the previous block state + + Called on self.open() + + :param block_height: the height of block to rollback to + :return: the height of block which is rollback to + """ + Logger.debug(tag=TAG, msg=f"rollback() start: BH={block_height}") + + if block_height < 0: + Logger.debug(tag=TAG, msg="rollback() end") + return -1, False + + path: str = self._get_backup_file_path(block_height) + if not os.path.isfile(path): + Logger.info(tag=TAG, msg=f"backup state file not found: {path}") + return -1, False + + block_height = -1 + is_calc_period_end_block = False + reader = WriteAheadLogReader() + + try: + reader.open(path) + is_calc_period_end_block = \ + bool(WALBackupState(reader.state) & WALBackupState.CALC_PERIOD_END_BLOCK) + + if reader.log_count == 2: + self._rollback_rc_db(reader, is_calc_period_end_block) + self._rollback_state_db(reader) + block_height = reader.block.height + + except BaseException as e: + Logger.debug(tag=TAG, msg=str(e)) + finally: + reader.close() + + # Remove backup file after rollback is done + self._remove_backup_file(path) + + Logger.debug(tag=TAG, msg=f"rollback() end: return={block_height}") + + return block_height, is_calc_period_end_block + + def _get_backup_file_path(self, block_height: int) -> str: + """ + + :param block_height: the height of block to rollback to + :return: backup state file path + """ + assert block_height >= 0 + + filename = get_backup_filename(block_height) + return os.path.join(self._root_path, filename) + + def _rollback_rc_db(self, reader: 'WriteAheadLogReader', is_calc_period_end_block: bool): + if is_calc_period_end_block: + Logger.info(tag=TAG, msg=f"BH-{reader.block.height} is a calc period end block") + self._rollback_rc_db_on_end_block(reader) + else: + db: 'KeyValueDatabase' = RewardCalcStorage.create_current_db(self._rc_data_path) + db.write_batch(reader.get_iterator(self._RC_DB)) + db.close() + + def _rollback_rc_db_on_end_block(self, reader: 'WriteAheadLogReader'): + current_rc_db_path, iiss_rc_db_path = RewardCalcStorage.scan_rc_db(self._rc_data_path) + + current_rc_db_exists = len(current_rc_db_path) > 0 + iiss_rc_db_exists = len(iiss_rc_db_path) > 0 + + if current_rc_db_exists: + if iiss_rc_db_exists: + # Remove the next calc_period current_rc_db and rename iiss_rc_db to current_rc_db + shutil.rmtree(current_rc_db_path) + os.rename(iiss_rc_db_path, current_rc_db_path) + else: + if iiss_rc_db_exists: + # iiss_rc_db -> current_rc_db + os.rename(iiss_rc_db_path, current_rc_db_path) + else: + # If both current_rc_db and iiss_rc_db do not exist, raise error + raise DatabaseException(f"RC DB not found") + + self._remove_block_produce_info(current_rc_db_path, reader.block.height) + + def _rollback_state_db(self, reader: 'WriteAheadLogReader'): + self._icx_db.write_batch(reader.get_iterator(self._STATE_DB)) + + def _clear_backup_files(self): + try: + with os.scandir(self._root_path) as it: + for entry in it: + if entry.is_file() \ + and entry.name.startswith("block-") \ + and entry.name.endswith(".bak"): + path = os.path.join(self._root_path, entry.name) + self._remove_backup_file(path) + except BaseException as e: + Logger.info(tag=TAG, msg=str(e)) + + @classmethod + def _remove_backup_file(cls, path: str): + try: + os.remove(path) + except FileNotFoundError: + pass + except BaseException as e: + Logger.debug(tag=TAG, msg=str(e)) + + @classmethod + def _remove_block_produce_info(cls, db_path: str, block_height: int): + """Remove block_produce_info of calc_period_end_block from current_db + + :param db_path: + :param block_height: + :return: + """ + key: bytes = make_block_produce_info_key(block_height) + + db = KeyValueDatabase.from_path(db_path, create_if_missing=False) + db.delete(key) + db.close() diff --git a/iconservice/icon_constant.py b/iconservice/icon_constant.py index 38d75951a..977189e55 100644 --- a/iconservice/icon_constant.py +++ b/iconservice/icon_constant.py @@ -26,6 +26,8 @@ IISS_LOG_TAG = "IISS" STEP_LOG_TAG = "STEP" WAL_LOG_TAG = "WAL" +ROLLBACK_LOG_TAG = "ROLLBACK" +BACKUP_LOG_TAG = "BACKUP" JSONRPC_VERSION = '2.0' CHARSET_ENCODING = 'utf-8' diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 25df620a1..4bd21851d 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, List, Any, Optional, Tuple from iconcommons.logger import Logger +from iconservice.database.backup_manager import BackupManager from .base.address import Address, generate_score_address, generate_score_address_for_tbears from .base.address import ZERO_SCORE_ADDRESS, GOVERNANCE_SCORE_ADDRESS from .base.block import Block, EMPTY_BLOCK @@ -28,6 +29,7 @@ DatabaseException) from .base.message import Message from .base.transaction import Transaction +from .base.type_converter_templates import ConstantKeys from .database.factory import ContextDatabaseFactory from .database.wal import WriteAheadLogReader from .database.wal import WriteAheadLogWriter, IissWAL, StateWAL, WALState @@ -66,7 +68,6 @@ from .utils import sha3_256, int_to_bytes, ContextEngine, ContextStorage from .utils import to_camel_case, bytes_to_hex from .utils.bloom import BloomFilter -from .base.type_converter_templates import ConstantKeys if TYPE_CHECKING: from .iconscore.icon_score_event_log import EventLog @@ -98,6 +99,7 @@ def __init__(self): self._context_factory = None self._state_db_root_path: Optional[str] = None self._wal_reader: Optional['WriteAheadLogReader'] = None + self._backup_manager: Optional[BackupManager] = None # JSON-RPC handlers self._handlers = { @@ -141,6 +143,7 @@ def open(self, conf: 'IconConfig'): self._deposit_handler = DepositHandler() self._icon_pre_validator = IconPreValidator() + self._backup_manager = BackupManager(state_db_root_path, rc_data_path, self._icx_context_db.key_value_db) IconScoreClassLoader.init(score_root_path) IconScoreContext.score_root_path = score_root_path @@ -1826,6 +1829,14 @@ def _commit_after_iiss(self, self._process_wal(context, precommit_data, is_calc_period_start_block, instant_block_hash) wal_writer.flush() + # Backup the previous block state + self._backup_manager.run(context=context, + prev_block=self._get_last_block(), + precommit_data=precommit_data, + iiss_wal=iiss_wal, + is_calc_period_start_block=is_calc_period_start_block, + instant_block_hash=instant_block_hash) + # Write iiss_wal to rc_db standby_db_info: Optional['RewardCalcDBInfo'] = \ self._process_iiss_commit(context, precommit_data, iiss_wal, is_calc_period_start_block) diff --git a/iconservice/iiss/reward_calc/msg_data.py b/iconservice/iiss/reward_calc/msg_data.py index ee63d15d2..7cb455821 100644 --- a/iconservice/iiss/reward_calc/msg_data.py +++ b/iconservice/iiss/reward_calc/msg_data.py @@ -171,6 +171,10 @@ def __str__(self): return info +def make_block_produce_info_key(block_height: int) -> bytes: + return BlockProduceInfoData.PREFIX + block_height.to_bytes(8, byteorder=DATA_BYTE_ORDER) + + class BlockProduceInfoData(Data): PREFIX = b'BP' @@ -183,8 +187,7 @@ def __init__(self): self.block_validator_list: Optional[List['Address']] = None def make_key(self) -> bytes: - block_height: bytes = self.block_height.to_bytes(8, byteorder=DATA_BYTE_ORDER) - return self.PREFIX + block_height + return make_block_produce_info_key(self.block_height) def make_value(self) -> bytes: data = [ From 6b7f808e003c3f843c8d148fa3b137142f2195b1 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Thu, 7 Nov 2019 16:13:29 +0900 Subject: [PATCH 03/36] Implement rollback() in IconServiceEngine and PRepEngine --- iconservice/icon_service_engine.py | 14 +++++++++- iconservice/prep/engine.py | 42 ++++++++++++++++++++++-------- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 4bd21851d..ce69212c9 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -19,6 +19,7 @@ from iconcommons.logger import Logger from iconservice.database.backup_manager import BackupManager +from iconservice.database.rollback_manager import RollbackManager from .base.address import Address, generate_score_address, generate_score_address_for_tbears from .base.address import ZERO_SCORE_ADDRESS, GOVERNANCE_SCORE_ADDRESS from .base.block import Block, EMPTY_BLOCK @@ -64,7 +65,7 @@ from .precommit_data_manager import PrecommitData, PrecommitDataManager, PrecommitFlag from .prep import PRepEngine, PRepStorage from .prep.data import PRep -from .utils import print_log_with_level, bytes_to_hex +from .utils import print_log_with_level from .utils import sha3_256, int_to_bytes, ContextEngine, ContextStorage from .utils import to_camel_case, bytes_to_hex from .utils.bloom import BloomFilter @@ -98,6 +99,7 @@ def __init__(self): self._deposit_handler = None self._context_factory = None self._state_db_root_path: Optional[str] = None + self._rc_data_path: Optional[str] = None self._wal_reader: Optional['WriteAheadLogReader'] = None self._backup_manager: Optional[BackupManager] = None @@ -136,6 +138,7 @@ def open(self, conf: 'IconConfig'): # Share one context db with all SCORE ContextDatabaseFactory.open(state_db_root_path, ContextDatabaseFactory.Mode.SINGLE_DB) self._state_db_root_path = state_db_root_path + self._rc_data_path = rc_data_path self._icx_context_db = ContextDatabaseFactory.create_by_name(ICON_DEX_DB_NAME) self._step_counter_factory = IconScoreStepCounterFactory() @@ -1988,6 +1991,15 @@ def remove_precommit_state(self, block_height: int, instant_block_hash: bytes) - def rollback(self, block_height: int, block_hash: bytes) -> dict: Logger.warning(tag=self.TAG, msg=f"rollback() start: height={block_height}, hash={bytes_to_hex(block_hash)}") + context = self._context_factory.create(IconScoreContextType.DIRECT, block=self._get_last_block()) + + # Rollback state_db and rc_data_db to those of a given block_height + rollback_manager = RollbackManager(self._state_db_root_path, self._rc_data_path) + rollback_manager.run(block_height) + + # Rollback preps and term + context.engine.prep.rollback(context) + response = { ConstantKeys.BLOCK_HEIGHT: block_height, ConstantKeys.BLOCK_HASH: block_hash diff --git a/iconservice/prep/engine.py b/iconservice/prep/engine.py index 69d8ffa76..83679adb6 100644 --- a/iconservice/prep/engine.py +++ b/iconservice/prep/engine.py @@ -28,7 +28,7 @@ from ..base.type_converter_templates import ConstantKeys from ..icon_constant import IISS_MAX_DELEGATIONS, Revision, IISS_MIN_IREP, PREP_PENALTY_SIGNATURE, \ PenaltyReason, TermFlag -from ..icon_constant import PRepGrade, PRepResultState, PRepStatus +from ..icon_constant import PRepGrade, PRepResultState, PRepStatus, ROLLBACK_LOG_TAG from ..iconscore.icon_score_context import IconScoreContext from ..iconscore.icon_score_event_log import EventLogEmitter from ..icx.icx_account import Account @@ -72,7 +72,7 @@ def __init__(self): "getInactivePReps": self.handle_get_inactive_preps } - self.preps = PRepContainer() + self._preps: Optional['PRepContainer'] = None # self.term should be None before decentralization self.term: Optional['Term'] = None self._initial_irep: Optional[int] = None @@ -80,6 +80,10 @@ def __init__(self): Logger.debug(tag=_TAG, msg="PRepEngine.__init__() end") + @property + def preps(self) -> Optional['PRepContainer']: + return self._preps + def open(self, context: 'IconScoreContext', term_period: int, @@ -93,7 +97,8 @@ def open(self, low_productivity_penalty_threshold, block_validation_penalty_threshold) - self._load_preps(context) + self._preps = self._load_preps(context) + self.load_term(context) self._initial_irep = irep context.engine.iiss.add_listener(self) @@ -117,12 +122,14 @@ def _init_penalty_imposer(self, low_productivity_penalty_threshold, block_validation_penalty_threshold) - def _load_preps(self, context: 'IconScoreContext'): - """Load a prep from db + @classmethod + def _load_preps(cls, context: 'IconScoreContext') -> 'PRepContainer': + """Load preps from state db - :return: + :return: new prep container instance """ icx_storage: 'IcxStorage' = context.storage.icx + preps = PRepContainer() for prep in context.storage.prep.get_prep_iterator(): account: 'Account' = icx_storage.get_account(context, prep.address, Intent.ALL) @@ -130,9 +137,10 @@ def _load_preps(self, context: 'IconScoreContext'): prep.stake = account.stake prep.delegated = account.delegated_amount - self.preps.add(prep) + preps.add(prep) - self.preps.freeze() + preps.freeze() + return preps def close(self): IconScoreContext.engine.iiss.remove_listener(self) @@ -161,7 +169,7 @@ def commit(self, _context: 'IconScoreContext', precommit_data: 'PrecommitData'): :return: """ # Updated every block - self.preps = precommit_data.preps + self._preps = precommit_data.preps # Exchange a term instance for some reasons: # - penalty for elected P-Reps(main, sub) @@ -169,8 +177,20 @@ def commit(self, _context: 'IconScoreContext', precommit_data: 'PrecommitData'): if precommit_data.term is not None: self.term: 'Term' = precommit_data.term - def rollback(self): - pass + def rollback(self, context: 'IconScoreContext'): + """After rollback is called, the state of prep_engine is reverted to that of a given block + + :param context: + :param _block_height: the height of the block to go back + :param _block_hash: + :return: + """ + Logger.info(tag=ROLLBACK_LOG_TAG, msg="rollback() start") + + self._preps = self._load_preps(context) + self.term = context.storage.prep.get_term(context) + + Logger.info(tag=ROLLBACK_LOG_TAG, msg="rollback() end") def on_block_invoked( self, From 8ea17e91325eadce557f4b4bfc3a8050f511fdd5 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Fri, 8 Nov 2019 15:50:05 +0900 Subject: [PATCH 04/36] Implement IconServiceEngine.rollback() --- iconservice/icon_service_engine.py | 13 ++++++++++++- iconservice/iconscore/icon_score_mapper.py | 7 +++++++ iconservice/iiss/reward_calc/storage.py | 4 ++++ iconservice/prep/engine.py | 12 ++++-------- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index ce69212c9..c8dad34d5 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -15,7 +15,7 @@ import os from copy import deepcopy -from typing import TYPE_CHECKING, List, Any, Optional, Tuple +from typing import TYPE_CHECKING, List, Any, Optional, Tuple, Dict, Union from iconcommons.logger import Logger from iconservice.database.backup_manager import BackupManager @@ -102,6 +102,7 @@ def __init__(self): self._rc_data_path: Optional[str] = None self._wal_reader: Optional['WriteAheadLogReader'] = None self._backup_manager: Optional[BackupManager] = None + self._conf: Optional[Dict[str, Union[str, int]]] = None # JSON-RPC handlers self._handlers = { @@ -196,6 +197,9 @@ def open(self, conf: 'IconConfig'): context, Address.from_string(conf[ConfigKey.BUILTIN_SCORE_OWNER])) self._init_global_value_by_governance_score(context) + # DO NOT change the values in conf + self._conf = conf + def _init_component_context(self): engine: 'ContextEngine' = ContextEngine(deploy=DeployEngine(), fee=FeeEngine(), @@ -308,6 +312,8 @@ def _make_service_flag(flag_table: dict) -> int: return make_flag def _load_builtin_scores(self, context: 'IconScoreContext', builtin_score_owner: 'Address'): + context.icon_score_mapper.clear() + current_address: 'Address' = context.current_address context.current_address = GOVERNANCE_SCORE_ADDRESS @@ -1997,6 +2003,11 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: rollback_manager = RollbackManager(self._state_db_root_path, self._rc_data_path) rollback_manager.run(block_height) + # Reload iconscores + builtin_score_owner: 'Address' = Address.from_string(self._conf[ConfigKey.BUILTIN_SCORE_OWNER]) + self._load_builtin_scores(context, builtin_score_owner) + self._init_global_value_by_governance_score(context) + # Rollback preps and term context.engine.prep.rollback(context) diff --git a/iconservice/iconscore/icon_score_mapper.py b/iconservice/iconscore/icon_score_mapper.py index d620e599b..b142ef23f 100644 --- a/iconservice/iconscore/icon_score_mapper.py +++ b/iconservice/iconscore/icon_score_mapper.py @@ -85,6 +85,13 @@ def update(self, mapper: 'IconScoreMapper'): with self._lock: self._score_mapper.update(mapper._score_mapper) + def clear(self): + if self._lock is None: + self._score_mapper.clear() + else: + with self._lock: + self._score_mapper.clear() + def close(self): for _, score_info in self._score_mapper.items(): score_info.score_db.close() diff --git a/iconservice/iiss/reward_calc/storage.py b/iconservice/iiss/reward_calc/storage.py index f8d556aa6..52c85d4dd 100644 --- a/iconservice/iiss/reward_calc/storage.py +++ b/iconservice/iiss/reward_calc/storage.py @@ -92,6 +92,10 @@ def open(self, context: IconScoreContext, path: str): # todo: check side effect of WAL self._supplement_db(context, revision) + @property + def key_value_db(self) -> 'KeyValueDatabase': + return self._db + def _supplement_db(self, context: 'IconScoreContext', revision: int): # Supplement db which is made by previous icon service version (as there is no version, revision and header) if revision < Revision.IISS.value: diff --git a/iconservice/prep/engine.py b/iconservice/prep/engine.py index 83679adb6..a1494c8d5 100644 --- a/iconservice/prep/engine.py +++ b/iconservice/prep/engine.py @@ -72,7 +72,7 @@ def __init__(self): "getInactivePReps": self.handle_get_inactive_preps } - self._preps: Optional['PRepContainer'] = None + self.preps = PRepContainer() # self.term should be None before decentralization self.term: Optional['Term'] = None self._initial_irep: Optional[int] = None @@ -80,10 +80,6 @@ def __init__(self): Logger.debug(tag=_TAG, msg="PRepEngine.__init__() end") - @property - def preps(self) -> Optional['PRepContainer']: - return self._preps - def open(self, context: 'IconScoreContext', term_period: int, @@ -97,7 +93,7 @@ def open(self, low_productivity_penalty_threshold, block_validation_penalty_threshold) - self._preps = self._load_preps(context) + self.preps = self._load_preps(context) self.load_term(context) self._initial_irep = irep @@ -169,7 +165,7 @@ def commit(self, _context: 'IconScoreContext', precommit_data: 'PrecommitData'): :return: """ # Updated every block - self._preps = precommit_data.preps + self.preps = precommit_data.preps # Exchange a term instance for some reasons: # - penalty for elected P-Reps(main, sub) @@ -187,7 +183,7 @@ def rollback(self, context: 'IconScoreContext'): """ Logger.info(tag=ROLLBACK_LOG_TAG, msg="rollback() start") - self._preps = self._load_preps(context) + self.preps = self._load_preps(context) self.term = context.storage.prep.get_term(context) Logger.info(tag=ROLLBACK_LOG_TAG, msg="rollback() end") From a917545e816a356ab825dc396e3c5e6231b73583 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Fri, 8 Nov 2019 16:44:26 +0900 Subject: [PATCH 05/36] Fix unittest errors after rebasing * Make storages ready prior to engines * Remove _post_component_context() from IconServiceEngine * Replace IconServiceEngine.rollback() with remove_precommit_state() * Fix all unittest errors --- iconservice/icon_service_engine.py | 40 +++++++++++++++--------------- iconservice/prep/engine.py | 6 +---- tests/test_icon_service_engine.py | 2 +- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index c8dad34d5..66901e053 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -191,8 +191,6 @@ def open(self, conf: 'IconConfig'): conf[ConfigKey.IPC_TIMEOUT], conf[ConfigKey.ICON_RC_DIR_PATH]) - self._post_open_component_context(context) - self._load_builtin_scores( context, Address.from_string(conf[ConfigKey.BUILTIN_SCORE_OWNER])) self._init_global_value_by_governance_score(context) @@ -241,6 +239,15 @@ def _open_component_context(cls, block_validation_penalty_threshold: int, ipc_timeout: int, icon_rc_path: str): + # storages MUST be prepared prior to engines because engines use them on open() + IconScoreContext.storage.deploy.open(context) + IconScoreContext.storage.fee.open(context) + IconScoreContext.storage.icx.open(context) + IconScoreContext.storage.iiss.open(context, iiss_meta_data, calc_period) + IconScoreContext.storage.prep.open(context, prep_reg_fee) + IconScoreContext.storage.issue.open(context) + IconScoreContext.storage.meta.open(context) + IconScoreContext.storage.rc.open(context, rc_data_path) IconScoreContext.engine.deploy.open(context) IconScoreContext.engine.fee.open(context) @@ -259,18 +266,6 @@ def _open_component_context(cls, block_validation_penalty_threshold) IconScoreContext.engine.issue.open(context) - IconScoreContext.storage.deploy.open(context) - IconScoreContext.storage.fee.open(context) - IconScoreContext.storage.icx.open(context) - IconScoreContext.storage.iiss.open(context, - iiss_meta_data, - calc_period) - IconScoreContext.storage.prep.open(context, - prep_reg_fee) - IconScoreContext.storage.issue.open(context) - IconScoreContext.storage.meta.open(context) - IconScoreContext.storage.rc.open(context, rc_data_path) - @classmethod def _close_component_context(cls, context: 'IconScoreContext'): IconScoreContext.engine.deploy.close() @@ -289,10 +284,6 @@ def _close_component_context(cls, context: 'IconScoreContext'): IconScoreContext.storage.meta.close(context) IconScoreContext.storage.rc.close() - @classmethod - def _post_open_component_context(cls, context: 'IconScoreContext'): - IconScoreContext.engine.prep.load_term(context) - @classmethod def get_ready_future(cls): return IconScoreContext.engine.iiss.get_ready_future() @@ -1995,15 +1986,24 @@ def remove_precommit_state(self, block_height: int, instant_block_hash: bytes) - Logger.warning(tag=self.TAG, msg="remove_precommit_state() end") def rollback(self, block_height: int, block_hash: bytes) -> dict: + """Rollback the current confirmed states to the old one indicated by block_height + + :param block_height: + :param block_hash: + :return: + """ Logger.warning(tag=self.TAG, msg=f"rollback() start: height={block_height}, hash={bytes_to_hex(block_hash)}") - context = self._context_factory.create(IconScoreContextType.DIRECT, block=self._get_last_block()) + last_block: 'Block' = self._get_last_block() + Logger.info(tag=self.TAG, msg=f"BH-{last_block.height} -> BH-{block_height}") + + context = self._context_factory.create(IconScoreContextType.DIRECT, block=last_block) # Rollback state_db and rc_data_db to those of a given block_height rollback_manager = RollbackManager(self._state_db_root_path, self._rc_data_path) rollback_manager.run(block_height) - # Reload iconscores + # Clear all iconscores and reload builtin scores only builtin_score_owner: 'Address' = Address.from_string(self._conf[ConfigKey.BUILTIN_SCORE_OWNER]) self._load_builtin_scores(context, builtin_score_owner) self._init_global_value_by_governance_score(context) diff --git a/iconservice/prep/engine.py b/iconservice/prep/engine.py index a1494c8d5..5195662f4 100644 --- a/iconservice/prep/engine.py +++ b/iconservice/prep/engine.py @@ -94,15 +94,11 @@ def open(self, block_validation_penalty_threshold) self.preps = self._load_preps(context) - self.load_term(context) + self.term = context.storage.prep.get_term(context) self._initial_irep = irep context.engine.iiss.add_listener(self) - def load_term(self, - context: 'IconScoreContext'): - self.term: Optional['Term'] = context.storage.prep.get_term(context) - def _init_penalty_imposer(self, penalty_grace_period: int, low_productivity_penalty_threshold: int, diff --git a/tests/test_icon_service_engine.py b/tests/test_icon_service_engine.py index 703773620..575268478 100644 --- a/tests/test_icon_service_engine.py +++ b/tests/test_icon_service_engine.py @@ -788,7 +788,7 @@ def test_rollback(self): self.assertIsInstance(block_result, list) self.assertEqual(state_root_hash, hashlib.sha3_256(b'').digest()) - self._engine.rollback(block.height, block.hash) + self._engine.remove_precommit_state(block.height, block.hash) self.assertIsNone(self._engine._precommit_data_manager.get(block.hash)) def test_invoke_v2_with_malformed_to_address_and_type_converter(self): From acb37bad37f94e0e2f261965efcf46ea32c613ab Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Fri, 8 Nov 2019 19:45:14 +0900 Subject: [PATCH 06/36] Implement ROLLBACK procotol between IS and RC --- iconservice/icon_service_engine.py | 3 ++ iconservice/iiss/engine.py | 16 +++++++- iconservice/iiss/reward_calc/ipc/message.py | 41 +++++++++++++++++++ .../iiss/reward_calc/ipc/reward_calc_proxy.py | 40 ++++++++++++++++++ 4 files changed, 98 insertions(+), 2 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 66901e053..2b681e498 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -2011,6 +2011,9 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: # Rollback preps and term context.engine.prep.rollback(context) + # Request reward calculator to rollback its db to the specific block_height + context.engine.iiss.rollback(context) + response = { ConstantKeys.BLOCK_HEIGHT: block_height, ConstantKeys.BLOCK_HASH: block_hash diff --git a/iconservice/iiss/engine.py b/iconservice/iiss/engine.py index b87ca1172..2d1312ec3 100644 --- a/iconservice/iiss/engine.py +++ b/iconservice/iiss/engine.py @@ -26,8 +26,10 @@ from ..base.ComponentBase import EngineBase from ..base.address import Address from ..base.address import ZERO_SCORE_ADDRESS -from ..base.exception import \ - InvalidParamsException, InvalidRequestException, OutOfBalanceException, FatalException +from ..base.exception import ( + InvalidParamsException, InvalidRequestException, + OutOfBalanceException, FatalException, InternalServiceErrorException +) from ..base.type_converter import TypeConverter from ..base.type_converter_templates import ConstantKeys, ParamType from ..icon_constant import IISS_MAX_DELEGATIONS, ISCORE_EXCHANGE_RATE, IISS_MAX_REWARD_RATE, \ @@ -906,3 +908,13 @@ def get_start_block_of_calc(cls, context: 'IconScoreContext') -> int: if end_block_height is not None and period is not None: start_calc_block: int = end_block_height - period + 1 return start_calc_block + + def rollback(self, _context: 'IconScoreContext', block_height: int, block_hash: bytes): + _status, _height, _hash = self._reward_calc_proxy.rollback(block_height, block_hash) + if _status and _height == block_height and _hash == block_hash: + return + + raise InternalServiceErrorException( + "rollback is failed: " + f"expected(True, {block_height}, {bytes_to_hex(block_hash)}) != " + f"actual({_status}, {_height}, {bytes_to_hex(_hash)})") diff --git a/iconservice/iiss/reward_calc/ipc/message.py b/iconservice/iiss/reward_calc/ipc/message.py index d6fee7ba6..7b4753a7c 100644 --- a/iconservice/iiss/reward_calc/ipc/message.py +++ b/iconservice/iiss/reward_calc/ipc/message.py @@ -486,6 +486,47 @@ def from_list(items: list) -> 'InitResponse': return InitResponse(msg_id, success, block_height) +class RollbackRequest(Request): + def __init__(self, block_height: int, block_hash: bytes): + super().__init__(MessageType.ROLLBACK) + + self.block_height = block_height + self.block_hash = block_hash + + def __str__(self): + return f"{self.msg_type.name}({self.msg_id}, " \ + f"{self.block_height}, {bytes_to_hex(self.block_hash)})" + + def _to_list(self) -> tuple: + return self.msg_type, self.msg_id, (self.block_height, self.block_hash) + + +class RollbackResponse(Response): + MSG_TYPE = MessageType.ROLLBACK + + def __init__(self, msg_id: int, status: bool, block_height: int, block_hash: bytes): + super().__init__() + + self.msg_id: int = msg_id + self.status: bool = status + self.block_height: int = block_height + self.block_hash: bytes = block_hash + + def __str__(self): + return f"ROLLBACK({self.msg_id}, {self.block_height}, {bytes_to_hex(self.block_hash)})" + + @staticmethod + def from_list(items: list) -> 'RollbackResponse': + msg_id: int = items[1] + payload: list = items[2] + + status: bool = payload[0] + block_height: int = payload[1] + block_hash: bytes = payload[2] + + return RollbackResponse(msg_id, status, block_height, block_hash) + + class ReadyNotification(Response): MSG_TYPE = MessageType.READY diff --git a/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py b/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py index 85ffe2fb3..9a10ea17b 100644 --- a/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py +++ b/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py @@ -454,6 +454,46 @@ async def _init_reward_calculator(self, block_height: int): return future.result() + def rollback(self, block_height: int, block_hash: bytes) -> Tuple[bool, int, bytes]: + """Request reward calculator to rollback the DB of the reward calculator to the specific block height. + + Reward calculator DOES NOT process other messages while processing ROLLBACK message + + :param block_height: + :param block_hash: + :return: + """ + + Logger.debug( + tag=_TAG, + msg=f"rollback() start: block_height={block_height}, block_hash={bytes_to_hex(block_hash)}" + ) + + future: concurrent.futures.Future = asyncio.run_coroutine_threadsafe( + self._rollback(block_height, block_hash), self._loop) + + try: + response: 'RollbackResponse' = future.result(self._ipc_timeout) + except asyncio.TimeoutError: + future.cancel() + raise TimeoutException("rollback message to RewardCalculator has timed-out") + + Logger.debug(tag=_TAG, msg=f"rollback() end. response: {response}") + + return response.status, response.block_height, response.block_hash + + async def _rollback(self, block_height: int, block_hash: bytes) -> list: + Logger.debug(tag=_TAG, msg="_rollback() start") + + request = RollbackRequest(block_height, block_hash) + + future: asyncio.Future = self._message_queue.put(request) + await future + + Logger.debug(tag=_TAG, msg="_rollback() end") + + return future.result() + def ready_handler(self, response: 'Response'): Logger.debug(tag=_TAG, msg=f"ready_handler() start {response}") From 4c0b83aa29c35089d8c98ee706c10258226a0432 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 12 Nov 2019 17:55:09 +0900 Subject: [PATCH 07/36] Fix minor type hint bugs on RewardProxyCalc class --- .../iiss/reward_calc/ipc/reward_calc_proxy.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py b/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py index 9a10ea17b..68d817f0c 100644 --- a/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py +++ b/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py @@ -132,7 +132,7 @@ def get_version(self): return response.version - async def _get_version(self): + async def _get_version(self) -> 'VersionResponse': Logger.debug(tag=_TAG, msg="_get_version() start") request = VersionRequest() @@ -166,7 +166,7 @@ def calculate(self, db_path: str, block_height: int) -> int: Logger.debug(tag=_TAG, msg=f"calculate() end: {response}") return response.status - async def _calculate(self, db_path: str, block_height: int): + async def _calculate(self, db_path: str, block_height: int) -> 'CalculateResponse': Logger.debug(tag=_TAG, msg="_calculate() start") request = CalculateRequest(db_path, block_height) @@ -284,7 +284,7 @@ async def _commit_claim(self, success: bool, address: 'Address', return future.result() - def query_iscore(self, address: 'Address') -> tuple: + def query_iscore(self, address: 'Address') -> Tuple[int, int]: """Returns the I-Score of a given address It should be called on query thread @@ -310,7 +310,7 @@ def query_iscore(self, address: 'Address') -> tuple: return response.iscore, response.block_height - async def _query_iscore(self, address: 'Address') -> list: + async def _query_iscore(self, address: 'Address') -> 'QueryResponse': """ :param address: @@ -343,7 +343,7 @@ def query_calculate_status(self) -> tuple: return response.status, response.block_height - async def _query_calculate_status(self) -> list: + async def _query_calculate_status(self) -> 'QueryCalculateStatusResponse': Logger.debug(tag=_TAG, msg="_query_calculate_status() start") request = QueryCalculateStatusRequest() @@ -371,7 +371,7 @@ def query_calculate_result(self, block_height) -> tuple: return response.status, response.block_height, response.iscore, response.state_hash - async def _query_calculate_result(self, block_height) -> list: + async def _query_calculate_result(self, block_height) -> 'QueryCalculateResultResponse': Logger.debug(tag=_TAG, msg="_query_calculate_result() start") request = QueryCalculateResultRequest(block_height) @@ -414,7 +414,7 @@ def commit_block(self, success: bool, block_height: int, block_hash: bytes) -> t return response.success, response.block_height, response.block_hash - async def _commit_block(self, success: bool, block_height: int, block_hash: bytes) -> list: + async def _commit_block(self, success: bool, block_height: int, block_hash: bytes) -> 'CommitBlockResponse': Logger.debug(tag=_TAG, msg="_commit_block() start") request = CommitBlockRequest(success, block_height, block_hash) @@ -482,7 +482,7 @@ def rollback(self, block_height: int, block_hash: bytes) -> Tuple[bool, int, byt return response.status, response.block_height, response.block_hash - async def _rollback(self, block_height: int, block_hash: bytes) -> list: + async def _rollback(self, block_height: int, block_hash: bytes) -> 'RollbackResponse': Logger.debug(tag=_TAG, msg="_rollback() start") request = RollbackRequest(block_height, block_hash) From be25485c9022b7c6bb17f2b0dc1c3e2c3cb1b844 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Thu, 14 Nov 2019 22:03:50 +0900 Subject: [PATCH 08/36] Before implementing IconServiceEngine.rollback() * Pass icx_db to RollbackManager in IconServiceEngine. * Add WALDBType * Rename "status" to "success" in RollbackResponse --- iconservice/database/rollback_manager.py | 33 +++++++++++++++---- iconservice/database/wal.py | 10 +++++- iconservice/icon_service_engine.py | 4 ++- iconservice/iiss/reward_calc/ipc/message.py | 8 ++--- .../iiss/reward_calc/ipc/reward_calc_proxy.py | 2 +- 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/iconservice/database/rollback_manager.py b/iconservice/database/rollback_manager.py index cb20c5701..67801f79c 100644 --- a/iconservice/database/rollback_manager.py +++ b/iconservice/database/rollback_manager.py @@ -15,22 +15,28 @@ import os import shutil -from typing import Tuple +from typing import TYPE_CHECKING, Tuple from iconcommons.logger import Logger + from .backup_manager import WALBackupState, get_backup_filename from .db import KeyValueDatabase -from .wal import WriteAheadLogReader +from .wal import WriteAheadLogReader, WALDBType from ..base.exception import DatabaseException from ..icon_constant import ROLLBACK_LOG_TAG from ..iiss.reward_calc import RewardCalcStorage from ..iiss.reward_calc.msg_data import make_block_produce_info_key +if TYPE_CHECKING: + from .db import KeyValueDatabase + + TAG = ROLLBACK_LOG_TAG class RollbackManager(object): - def __init__(self, state_db_root_path: str, rc_data_path: str): + def __init__(self, icx_db: 'KeyValueDatabase', state_db_root_path: str, rc_data_path: str): + self._icx_db: 'KeyValueDatabase' = icx_db self._rc_data_path = rc_data_path self._root_path = os.path.join(state_db_root_path, "backup") @@ -91,16 +97,31 @@ def _get_backup_file_path(self, block_height: int) -> str: return os.path.join(self._root_path, filename) def _rollback_rc_db(self, reader: 'WriteAheadLogReader', is_calc_period_end_block: bool): + """Rollback the state of rc_db to the previous one + + :param reader: + :param is_calc_period_end_block: + :return: + """ if is_calc_period_end_block: Logger.info(tag=TAG, msg=f"BH-{reader.block.height} is a calc period end block") self._rollback_rc_db_on_end_block(reader) else: db: 'KeyValueDatabase' = RewardCalcStorage.create_current_db(self._rc_data_path) - db.write_batch(reader.get_iterator(self._RC_DB)) + db.write_batch(reader.get_iterator(WALDBType.RC.value)) db.close() def _rollback_rc_db_on_end_block(self, reader: 'WriteAheadLogReader'): - current_rc_db_path, iiss_rc_db_path = RewardCalcStorage.scan_rc_db(self._rc_data_path) + """ + + :param reader: + :return: + """ + + current_rc_db_path, standby_rc_db_path, iiss_rc_db_path = \ + RewardCalcStorage.scan_rc_db(self._rc_data_path) + # Assume that standby_rc_db does not exist + assert standby_rc_db_path == "" current_rc_db_exists = len(current_rc_db_path) > 0 iiss_rc_db_exists = len(iiss_rc_db_path) > 0 @@ -121,7 +142,7 @@ def _rollback_rc_db_on_end_block(self, reader: 'WriteAheadLogReader'): self._remove_block_produce_info(current_rc_db_path, reader.block.height) def _rollback_state_db(self, reader: 'WriteAheadLogReader'): - self._icx_db.write_batch(reader.get_iterator(self._STATE_DB)) + self._icx_db.write_batch(reader.get_iterator(WALDBType.STATE.value)) def _clear_backup_files(self): try: diff --git a/iconservice/database/wal.py b/iconservice/database/wal.py index cd09b041f..2c97a240b 100644 --- a/iconservice/database/wal.py +++ b/iconservice/database/wal.py @@ -19,12 +19,15 @@ from ..iiss.reward_calc.storage import Storage, get_rc_version from ..utils.msgpack_for_db import MsgPackForDB -__all__ = ("WriteAheadLogWriter", "WriteAheadLogReader", "WALogable", "StateWAL", "IissWAL", "WALState") +__all__ = ( + "WriteAheadLogWriter", "WriteAheadLogReader", "WALogable", "StateWAL", "IissWAL", "WALState", "WALDBType" +) import struct from abc import ABCMeta from typing import Optional, Tuple, Iterable, List import os +from enum import Enum import msgpack from iconcommons.logger import Logger @@ -52,6 +55,11 @@ _OFFSET_LOG_START_OFFSETS = _OFFSET_LOG_COUNT + 4 +class WALDBType(Enum): + RC = 0 + STATE = 1 + + class WALState(Flag): CALC_PERIOD_START_BLOCK = auto() # Write WAL to rc_db diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 2b681e498..80cb1e61e 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, List, Any, Optional, Tuple, Dict, Union from iconcommons.logger import Logger + from iconservice.database.backup_manager import BackupManager from iconservice.database.rollback_manager import RollbackManager from .base.address import Address, generate_score_address, generate_score_address_for_tbears @@ -2000,7 +2001,8 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: context = self._context_factory.create(IconScoreContextType.DIRECT, block=last_block) # Rollback state_db and rc_data_db to those of a given block_height - rollback_manager = RollbackManager(self._state_db_root_path, self._rc_data_path) + rollback_manager = RollbackManager( + self._icx_context_db.key_value_db, self._state_db_root_path, self._rc_data_path) rollback_manager.run(block_height) # Clear all iconscores and reload builtin scores only diff --git a/iconservice/iiss/reward_calc/ipc/message.py b/iconservice/iiss/reward_calc/ipc/message.py index 7b4753a7c..07e8e7210 100644 --- a/iconservice/iiss/reward_calc/ipc/message.py +++ b/iconservice/iiss/reward_calc/ipc/message.py @@ -504,11 +504,11 @@ def _to_list(self) -> tuple: class RollbackResponse(Response): MSG_TYPE = MessageType.ROLLBACK - def __init__(self, msg_id: int, status: bool, block_height: int, block_hash: bytes): + def __init__(self, msg_id: int, success: bool, block_height: int, block_hash: bytes): super().__init__() self.msg_id: int = msg_id - self.status: bool = status + self.success: bool = success self.block_height: int = block_height self.block_hash: bytes = block_hash @@ -520,11 +520,11 @@ def from_list(items: list) -> 'RollbackResponse': msg_id: int = items[1] payload: list = items[2] - status: bool = payload[0] + success: bool = payload[0] block_height: int = payload[1] block_hash: bytes = payload[2] - return RollbackResponse(msg_id, status, block_height, block_hash) + return RollbackResponse(msg_id, success, block_height, block_hash) class ReadyNotification(Response): diff --git a/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py b/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py index 68d817f0c..fcbdaa398 100644 --- a/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py +++ b/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py @@ -480,7 +480,7 @@ def rollback(self, block_height: int, block_hash: bytes) -> Tuple[bool, int, byt Logger.debug(tag=_TAG, msg=f"rollback() end. response: {response}") - return response.status, response.block_height, response.block_hash + return response.success, response.block_height, response.block_hash async def _rollback(self, block_height: int, block_hash: bytes) -> 'RollbackResponse': Logger.debug(tag=_TAG, msg="_rollback() start") From ef170bca95512b99af1893be4b68fad879d855c7 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Mon, 18 Nov 2019 20:24:10 +0900 Subject: [PATCH 09/36] Implement IconServiceEngine.rollback() * Call rollback() of all storages * Call rollback() of all engines * Exception handling on rollback * Minor bugfix * Remove IDE warning messages --- iconservice/base/ComponentBase.py | 6 ++ iconservice/icon_inner_service.py | 8 +-- iconservice/icon_service_engine.py | 92 +++++++++++++++++++------ iconservice/icx/storage.py | 18 ++++- iconservice/iiss/engine.py | 22 ++++-- iconservice/iiss/reward_calc/storage.py | 22 +++++- iconservice/iiss/storage.py | 3 + iconservice/meta/storage.py | 6 +- iconservice/prep/engine.py | 2 +- 9 files changed, 140 insertions(+), 39 deletions(-) diff --git a/iconservice/base/ComponentBase.py b/iconservice/base/ComponentBase.py index b0edb51d1..f1ad24782 100644 --- a/iconservice/base/ComponentBase.py +++ b/iconservice/base/ComponentBase.py @@ -33,6 +33,9 @@ def open(self, *args, **kwargs): def close(self): pass + def rollback(self, context: 'IconScoreContext', block_height: int, block_hash: bytes): + pass + class StorageBase(ABC): @@ -54,3 +57,6 @@ def close(self, context: 'IconScoreContext'): if self._db: self._db.close(context) self._db = None + + def rollback(self, context: 'IconScoreContext', block_height: int, block_hash: bytes): + pass diff --git a/iconservice/icon_inner_service.py b/iconservice/icon_inner_service.py index ad02f1ae3..554506c73 100644 --- a/iconservice/icon_inner_service.py +++ b/iconservice/icon_inner_service.py @@ -67,8 +67,8 @@ def _check_icon_service_ready(self): @staticmethod def _log_exception(e: BaseException, tag: str = ICON_INNER_LOG_TAG) -> None: - Logger.exception(e, tag) - Logger.error(e, tag) + Logger.exception(str(e), tag) + Logger.error(str(e), tag) @message_queue_task async def hello(self): @@ -360,7 +360,7 @@ async def rollback(self, request: dict): self._check_icon_service_ready() if self._is_thread_flag_on(EnableThreadFlag.INVOKE): - loop = get_event_loop() + loop = asyncio.get_event_loop() response = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._rollback, request) else: response = self._rollback(request) @@ -427,7 +427,7 @@ def _validate_transaction(self, request: dict): return response @message_queue_task - async def change_block_hash(self, params): + async def change_block_hash(self, _params): self._check_icon_service_ready() diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 80cb1e61e..fb6caf25f 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -27,8 +27,7 @@ from .base.exception import ( ExceptionCode, IconServiceBaseException, ScoreNotFoundException, AccessDeniedException, IconScoreException, InvalidParamsException, InvalidBaseTransactionException, - MethodNotFoundException, - DatabaseException) + MethodNotFoundException, InternalServiceErrorException, DatabaseException) from .base.message import Message from .base.transaction import Transaction from .base.type_converter_templates import ConstantKeys @@ -42,7 +41,7 @@ ICON_DEX_DB_NAME, ICON_SERVICE_LOG_TAG, IconServiceFlag, ConfigKey, IISS_METHOD_TABLE, PREP_METHOD_TABLE, NEW_METHOD_TABLE, Revision, BASE_TRANSACTION_INDEX, IISS_DB, IISS_INITIAL_IREP, DEBUG_METHOD_TABLE, PREP_MAIN_PREPS, PREP_MAIN_AND_SUB_PREPS, - ISCORE_EXCHANGE_RATE, STEP_LOG_TAG, TERM_PERIOD, BlockVoteStatus, WAL_LOG_TAG) + ISCORE_EXCHANGE_RATE, STEP_LOG_TAG, TERM_PERIOD, BlockVoteStatus, WAL_LOG_TAG, ROLLBACK_LOG_TAG) from .iconscore.icon_pre_validator import IconPreValidator from .iconscore.icon_score_class_loader import IconScoreClassLoader from .iconscore.icon_score_context import IconScoreContext, IconScoreFuncType, ContextContainer, IconScoreContextFactory @@ -1993,12 +1992,54 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: :param block_hash: :return: """ - Logger.warning(tag=self.TAG, msg=f"rollback() start: height={block_height}, hash={bytes_to_hex(block_hash)}") + Logger.warning(tag=ROLLBACK_LOG_TAG, + msg=f"rollback() start: height={block_height} hash={bytes_to_hex(block_hash)}") last_block: 'Block' = self._get_last_block() - Logger.info(tag=self.TAG, msg=f"BH-{last_block.height} -> BH-{block_height}") + Logger.info(tag=self.TAG, msg=f"last_block={last_block}") - context = self._context_factory.create(IconScoreContextType.DIRECT, block=last_block) + # If rollback is impossible for the current status, + # self._is_rollback_needed() should raise an InternalServiceErrorException + try: + if self._is_rollback_needed(last_block, block_height, block_hash): + context = self._context_factory.create(IconScoreContextType.DIRECT, block=last_block) + self._rollback(context, block_height, block_hash) + except BaseException as e: + Logger.error(tag=ROLLBACK_LOG_TAG, msg=str(e)) + raise InternalServiceErrorException( + f"Failed to rollback: {last_block.height} -> {block_height}") + + response = { + ConstantKeys.BLOCK_HEIGHT: block_height, + ConstantKeys.BLOCK_HASH: block_hash + } + + Logger.warning(tag=ROLLBACK_LOG_TAG, + msg=f"rollback() end: height={block_height}, hash={bytes_to_hex(block_hash)}") + + return response + + @classmethod + def _is_rollback_needed(cls, last_block: 'Block', block_height: int, block_hash: bytes) -> bool: + """Check if rollback is needed + """ + if block_height == last_block.height + 1: + return True + if block_height == last_block.height and block_hash == last_block.hash: + return False + + raise InternalServiceErrorException( + f"Failed to rollback: " + f"height={block_height} " + f"hash={bytes_to_hex(block_hash)} " + f"last_block={last_block}") + + def _rollback(self, context: 'IconScoreContext', block_height: int, block_hash: bytes): + # Close storage + IconScoreContext.storage.rc.close() + + # Rollback the state of reward_calculator prior to iconservice + IconScoreContext.engine.iiss.rollback_reward_calculator(block_height, block_hash) # Rollback state_db and rc_data_db to those of a given block_height rollback_manager = RollbackManager( @@ -2010,20 +2051,31 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: self._load_builtin_scores(context, builtin_score_owner) self._init_global_value_by_governance_score(context) - # Rollback preps and term - context.engine.prep.rollback(context) - - # Request reward calculator to rollback its db to the specific block_height - context.engine.iiss.rollback(context) - - response = { - ConstantKeys.BLOCK_HEIGHT: block_height, - ConstantKeys.BLOCK_HASH: block_hash - } - - Logger.warning(tag=self.TAG, msg=f"rollback() end: height={block_height}, hash={bytes_to_hex(block_hash)}") - - return response + # Rollback storages + storages = [ + IconScoreContext.storage.deploy, + IconScoreContext.storage.fee, + IconScoreContext.storage.icx, + IconScoreContext.storage.iiss, + IconScoreContext.storage.prep, + IconScoreContext.storage.issue, + IconScoreContext.storage.meta, + IconScoreContext.storage.rc, + ] + for storage in storages: + storage.rollback(context, block_height, block_hash) + + # Rollback engines to block_height + engines = [ + IconScoreContext.engine.deploy, + IconScoreContext.engine.fee, + IconScoreContext.engine.icx, + IconScoreContext.engine.iiss, + IconScoreContext.engine.prep, + IconScoreContext.engine.issue, + ] + for engine in engines: + engine.rollback(context, block_height, block_hash) def clear_context_stack(self): """Clear IconScoreContext stacks diff --git a/iconservice/icx/storage.py b/iconservice/icx/storage.py index fa42a638a..c3962753c 100644 --- a/iconservice/icx/storage.py +++ b/iconservice/icx/storage.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Optional, Union from iconcommons import Logger + from .coin_part import CoinPart, CoinPartFlag, CoinPartType from .delegation_part import DelegationPart from .icx_account import Account @@ -26,7 +27,8 @@ from ..base.ComponentBase import StorageBase from ..base.address import Address from ..base.block import Block, EMPTY_BLOCK -from ..icon_constant import DEFAULT_BYTE_SIZE, DATA_BYTE_ORDER, ICX_LOG_TAG +from ..icon_constant import DEFAULT_BYTE_SIZE, DATA_BYTE_ORDER, ICX_LOG_TAG, ROLLBACK_LOG_TAG +from ..utils import bytes_to_hex if TYPE_CHECKING: from ..database.db import ContextDatabase @@ -68,13 +70,23 @@ def __init__(self, db: 'ContextDatabase'): super().__init__(db) self._db = db self._last_block = EMPTY_BLOCK - self._genesis: 'Address' = None - self._fee_treasury: 'Address' = None + self._genesis: Optional['Address'] = None + self._fee_treasury: Optional['Address'] = None def open(self, context: 'IconScoreContext'): self._load_special_address(context, self._GENESIS_DB_KEY) self._load_special_address(context, self._TREASURY_DB_KEY) + def rollback(self, context: 'IconScoreContext', block_height: int, block_hash: bytes): + Logger.info(tag=ROLLBACK_LOG_TAG, + msg=f"rollback() start: block_height={block_height} block_hash={bytes_to_hex(block_hash)}") + + self._load_special_address(context, self._GENESIS_DB_KEY) + self._load_special_address(context, self._TREASURY_DB_KEY) + self.load_last_block_info(context) + + Logger.info(tag=ROLLBACK_LOG_TAG, msg="rollback() end") + @property def last_block(self) -> 'Block': return self._last_block diff --git a/iconservice/iiss/engine.py b/iconservice/iiss/engine.py index 2d1312ec3..faaad2384 100644 --- a/iconservice/iiss/engine.py +++ b/iconservice/iiss/engine.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Any, Optional, List, Dict, Tuple, Union from iconcommons.logger import Logger + from .reward_calc.data_creator import DataCreator as RewardCalcDataCreator from .reward_calc.ipc.message import CalculateDoneNotification, ReadyNotification from .reward_calc.ipc.reward_calc_proxy import RewardCalcProxy @@ -33,7 +34,7 @@ from ..base.type_converter import TypeConverter from ..base.type_converter_templates import ConstantKeys, ParamType from ..icon_constant import IISS_MAX_DELEGATIONS, ISCORE_EXCHANGE_RATE, IISS_MAX_REWARD_RATE, \ - IconScoreContextType, IISS_LOG_TAG, RCCalculateResult, INVALID_CLAIM_TX, Revision + IconScoreContextType, IISS_LOG_TAG, ROLLBACK_LOG_TAG, RCCalculateResult, INVALID_CLAIM_TX, Revision from ..iconscore.icon_score_context import IconScoreContext from ..iconscore.icon_score_event_log import EventLogEmitter from ..icx import Intent @@ -909,12 +910,19 @@ def get_start_block_of_calc(cls, context: 'IconScoreContext') -> int: start_calc_block: int = end_block_height - period + 1 return start_calc_block - def rollback(self, _context: 'IconScoreContext', block_height: int, block_hash: bytes): + def rollback_reward_calculator(self, block_height: int, block_hash: bytes): + Logger.info(tag=ROLLBACK_LOG_TAG, + msg=f"rollback_reward_calculator() start: " + f"height={block_height} hash={bytes_to_hex(block_hash)}") + _status, _height, _hash = self._reward_calc_proxy.rollback(block_height, block_hash) + Logger.info(tag=ROLLBACK_LOG_TAG, + msg=f"RewardCalculator response: " + f"status={_status} height={_height} hash={bytes_to_hex(_hash)}") + if _status and _height == block_height and _hash == block_hash: - return + Logger.info(tag=ROLLBACK_LOG_TAG, + msg=f"rollback_reward_calculator() end: " + f"height={block_height} hash={bytes_to_hex(block_hash)}") - raise InternalServiceErrorException( - "rollback is failed: " - f"expected(True, {block_height}, {bytes_to_hex(block_hash)}) != " - f"actual({_status}, {_height}, {bytes_to_hex(_hash)})") + raise InternalServiceErrorException("Failed to rollback RewardCalculator") diff --git a/iconservice/iiss/reward_calc/storage.py b/iconservice/iiss/reward_calc/storage.py index 52c85d4dd..a52e88fff 100644 --- a/iconservice/iiss/reward_calc/storage.py +++ b/iconservice/iiss/reward_calc/storage.py @@ -19,14 +19,17 @@ from typing import TYPE_CHECKING, Optional, Tuple, List, Set from iconcommons import Logger + from ..reward_calc.msg_data import Header, TxData, PRepsData, TxType from ...base.exception import DatabaseException, InternalServiceErrorException from ...database.db import KeyValueDatabase from ...icon_constant import ( - DATA_BYTE_ORDER, Revision, RC_DATA_VERSION_TABLE, RC_DB_VERSION_0, IISS_LOG_TAG, WAL_LOG_TAG + DATA_BYTE_ORDER, Revision, RC_DATA_VERSION_TABLE, RC_DB_VERSION_0, + IISS_LOG_TAG, WAL_LOG_TAG, ROLLBACK_LOG_TAG ) from ...iconscore.icon_score_context import IconScoreContext from ...iiss.reward_calc.data_creator import DataCreator +from ...utils import bytes_to_hex from ...utils.msgpack_for_db import MsgPackForDB if TYPE_CHECKING: @@ -92,6 +95,23 @@ def open(self, context: IconScoreContext, path: str): # todo: check side effect of WAL self._supplement_db(context, revision) + def rollback(self, _context: 'IconScoreContext', block_height: int, block_hash: bytes): + Logger.info(tag=ROLLBACK_LOG_TAG, + msg=f"rollback() start: block_height={block_height} block_hash={bytes_to_hex(block_hash)}") + + if self._db is not None: + raise InternalServiceErrorException("current_db has been opened on rollback") + + if not os.path.exists(self._path): + raise DatabaseException(f"Invalid IISS DB path: {self._path}") + + self._db = self.create_current_db(self._path) + + self._db_iiss_tx_index = self._load_last_transaction_index() + Logger.info(tag=IISS_LOG_TAG, msg=f"last_transaction_index on open={self._db_iiss_tx_index}") + + Logger.info(tag=ROLLBACK_LOG_TAG, msg="rollback() end") + @property def key_value_db(self) -> 'KeyValueDatabase': return self._db diff --git a/iconservice/iiss/storage.py b/iconservice/iiss/storage.py index 7b1cca1a0..341dc22e9 100644 --- a/iconservice/iiss/storage.py +++ b/iconservice/iiss/storage.py @@ -58,6 +58,9 @@ def open(self, self._meta_data: 'IISSMetaData' = meta_data + def rollback(self, context: 'IconScoreContext', block_height: int, block_hash: bytes): + pass + @property def reward_min(self) -> int: return self._meta_data.reward_min diff --git a/iconservice/meta/storage.py b/iconservice/meta/storage.py index 58a8f9118..0bc5145b1 100644 --- a/iconservice/meta/storage.py +++ b/iconservice/meta/storage.py @@ -46,7 +46,7 @@ def get_last_calc_info(self, context: 'IconScoreContext') -> Tuple[int, int]: if value is None: return -1, -1 data: list = MsgPackForDB.loads(value) - version = data[0] + _version = data[0] return data[1], data[2] def put_last_term_info(self, @@ -62,7 +62,7 @@ def get_last_term_info(self, context: 'IconScoreContext') -> Tuple[int, int]: if value is None: return -1, -1 data: list = MsgPackForDB.loads(value) - version = data[0] + _version = data[0] return data[1], data[2] def put_last_main_preps(self, @@ -77,5 +77,5 @@ def get_last_main_preps(self, context: 'IconScoreContext') -> List['Address']: if value is None: return [] data: list = MsgPackForDB.loads(value) - version = data[0] + _version = data[0] return data[1] diff --git a/iconservice/prep/engine.py b/iconservice/prep/engine.py index 5195662f4..5e565b0e0 100644 --- a/iconservice/prep/engine.py +++ b/iconservice/prep/engine.py @@ -169,7 +169,7 @@ def commit(self, _context: 'IconScoreContext', precommit_data: 'PrecommitData'): if precommit_data.term is not None: self.term: 'Term' = precommit_data.term - def rollback(self, context: 'IconScoreContext'): + def rollback(self, context: 'IconScoreContext', _block_height: int, _block_hash: bytes): """After rollback is called, the state of prep_engine is reverted to that of a given block :param context: From 73f9cf417bbc47db38fbd31fb05316943f514a3d Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Mon, 18 Nov 2019 21:06:29 +0900 Subject: [PATCH 10/36] Add block_hash to ReadyNotification --- iconservice/iiss/engine.py | 3 ++ .../iiss/reward_calc/ipc/reward_calc_proxy.py | 28 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/iconservice/iiss/engine.py b/iconservice/iiss/engine.py index faaad2384..4b1ac8c9b 100644 --- a/iconservice/iiss/engine.py +++ b/iconservice/iiss/engine.py @@ -926,3 +926,6 @@ def rollback_reward_calculator(self, block_height: int, block_hash: bytes): f"height={block_height} hash={bytes_to_hex(block_hash)}") raise InternalServiceErrorException("Failed to rollback RewardCalculator") + + def get_reward_calculator_commit_block(self) -> Optional[Tuple[int, bytes]]: + return self._reward_calc_proxy.get_commit_block() diff --git a/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py b/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py index fcbdaa398..ef37eed20 100644 --- a/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py +++ b/iconservice/iiss/reward_calc/ipc/reward_calc_proxy.py @@ -37,6 +37,23 @@ _TAG = "RCP" +class RewardCalcBlock(object): + """Stores the latest commit status of reward calculator + """ + + def __init__(self, block_height: int, block_hash: bytes): + self._block_height = block_height + self._block_hash = block_hash + + @property + def block_height(self) -> int: + return self._block_height + + @property + def block_hash(self) -> bytes: + return self._block_hash + + class RewardCalcProxy(object): """Communicates with Reward Calculator through UNIX Domain Socket @@ -62,6 +79,7 @@ def __init__(self, self._calculate_done_callback: Optional[Callable] = calc_done_callback self._ipc_timeout = ipc_timeout self._icon_rc_path = icon_rc_path + self._rc_block: Optional[RewardCalcBlock] = None Logger.debug(tag=_TAG, msg="__init__() end") @@ -102,6 +120,7 @@ def close(self): self._message_queue = None self._loop = None + self._rc_block = None Logger.debug(tag=_TAG, msg="close() end") @@ -494,13 +513,14 @@ async def _rollback(self, block_height: int, block_hash: bytes) -> 'RollbackResp return future.result() - def ready_handler(self, response: 'Response'): + def ready_handler(self, response: 'ReadyNotification'): Logger.debug(tag=_TAG, msg=f"ready_handler() start {response}") if self._ready_callback is not None: self._ready_callback(response) self._ready_future.set_result(RCStatus.READY) + self._rc_block = RewardCalcBlock(response.block_height, response.block_height) def get_ready_future(self): return self._ready_future @@ -570,3 +590,9 @@ def stop_reward_calc(self): if self._reward_calc is not None: self._reward_calc.kill() self._reward_calc = None + + def get_commit_block(self) -> Optional[Tuple[int, bytes]]: + if self._rc_block is None: + return None + + return self._rc_block.block_height, self._rc_block.block_hash From b064b15c72d3bf232669c26ddf3bfc39b7e89259 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 19 Nov 2019 01:21:02 +0900 Subject: [PATCH 11/36] Unittest for BackupManager is under development --- iconservice/database/backup_manager.py | 19 ++--- iconservice/icon_service_engine.py | 5 +- tests/database/test_backup_manager.py | 109 +++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 tests/database/test_backup_manager.py diff --git a/iconservice/database/backup_manager.py b/iconservice/database/backup_manager.py index 5ee3d60ab..6e7ce4143 100644 --- a/iconservice/database/backup_manager.py +++ b/iconservice/database/backup_manager.py @@ -26,7 +26,6 @@ from .wal import IissWAL from ..base.block import Block from ..database.batch import BlockBatch - from ..iconscore.icon_score_context import IconScoreContext from ..precommit_data_manager import PrecommitData @@ -50,8 +49,6 @@ class BackupManager(object): """Backup and rollback for the previous block state """ - _RC_DB = 0 - _STATE_DB = 1 def __init__(self, state_db_root_path: str, rc_data_path: str, icx_db: 'KeyValueDatabase'): Logger.debug(tag=TAG, @@ -83,17 +80,19 @@ def _get_backup_file_path(self, block_height: int) -> str: return os.path.join(self._root_path, filename) def run(self, - context: 'IconScoreContext', + revision: int, + rc_db: 'KeyValueDatabase', prev_block: 'Block', - precommit_data: 'PrecommitData', + block_batch: 'BlockBatch', iiss_wal: 'IissWAL', is_calc_period_start_block: bool, instant_block_hash: bytes): """Backup the previous block state - :param context: + :param revision: + :param rc_db: :param prev_block: the latest confirmed block height during commit - :param precommit_data: + :param block_batch: :param iiss_wal: :param is_calc_period_start_block: :param instant_block_hash: @@ -107,14 +106,14 @@ def run(self, self._clear_backup_files() writer = WriteAheadLogWriter( - context.revision, max_log_count=2, block=prev_block, instant_block_hash=instant_block_hash) + revision, max_log_count=2, block=prev_block, instant_block_hash=instant_block_hash) writer.open(path) if is_calc_period_start_block: writer.write_state(WALBackupState.CALC_PERIOD_END_BLOCK.value) - self._backup_rc_db(writer, context.storage.rc.key_value_db, iiss_wal) - self._backup_state_db(writer, self._icx_db, precommit_data.block_batch) + self._backup_rc_db(writer, rc_db, iiss_wal) + self._backup_state_db(writer, self._icx_db, block_batch) writer.close() diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index fb6caf25f..ee6f4bf18 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -1830,9 +1830,10 @@ def _commit_after_iiss(self, wal_writer.flush() # Backup the previous block state - self._backup_manager.run(context=context, + self._backup_manager.run(revision=context.revision, + rc_db=context.storage.rc.key_value_db, prev_block=self._get_last_block(), - precommit_data=precommit_data, + block_batch=precommit_data.block_batch, iiss_wal=iiss_wal, is_calc_period_start_block=is_calc_period_start_block, instant_block_hash=instant_block_hash) diff --git a/tests/database/test_backup_manager.py b/tests/database/test_backup_manager.py new file mode 100644 index 000000000..8d9e43882 --- /dev/null +++ b/tests/database/test_backup_manager.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import shutil +import os +from collections import OrderedDict +import hashlib + +from iconservice.icon_constant import Revision +from iconservice.database.backup_manager import BackupManager +from iconservice.database.db import KeyValueDatabase +from iconservice.base.block import Block + + +def _create_db(path) -> 'KeyValueDatabase': + db = KeyValueDatabase.from_path(path, create_if_missing=True) + + for i in range(5): + postfix: bytes = i.to_bytes(1, "big", signed=False) + key: bytes = b"key" + postfix + value: bytes = b"value" + postfix + + db.put(key, value) + + return db + + +class TestBackupManager(unittest.TestCase): + def setUp(self) -> None: + state_db_root_path = "./test_backup_manager_db" + shutil.rmtree(state_db_root_path, ignore_errors=True) + + os.mkdir(state_db_root_path) + + state_db_path = os.path.join(state_db_root_path, "icon_dex") + rc_data_path = os.path.join(state_db_root_path, "iiss") + + icx_db = _create_db(state_db_path) + + self.backup_manager = BackupManager( + state_db_root_path=state_db_root_path, + rc_data_path=rc_data_path, + icx_db=icx_db + ) + + self._state_db_root_path = state_db_root_path + self._icx_db = icx_db + self.rc_db = KeyValueDatabase.from_path(rc_data_path) + + def tearDown(self) -> None: + if self._icx_db: + self._icx_db.close() + self._icx_db = None + + if self.rc_db: + self.rc_db.close() + self.rc_db = None + + shutil.rmtree(self._state_db_root_path, ignore_errors=True) + + def test_run(self): + backup_manager = self.backup_manager + block_hash: bytes = hashlib.sha3_256(b"block_hash").digest() + prev_hash: bytes = hashlib.sha3_256(b"prev_hash").digest() + instant_block_hash: bytes = hashlib.sha3_256(b"instant_block_hash").digest() + + revision = Revision.DECENTRALIZATION.value + rc_db = self.rc_db + last_block = Block( + block_height=100, + block_hash=block_hash, + timestamp=0, + prev_hash=prev_hash, + cumulative_fee=0 + ) + block_batch = OrderedDict() + block_batch[b"key0"] = b"new value0" + block_batch[b"key1"] = None + block_batch[b"key2"] = b"value2" + + is_calc_period_start_block = False + + rc_data = OrderedDict() + rc_data[b"key0"] = b"value0" + + def iiss_wal(): + for key in rc_data: + yield key, rc_data[key] + + backup_manager.run(revision=revision, + rc_db=rc_db, + prev_block=last_block, + block_batch=block_batch, + iiss_wal=iiss_wal(), + is_calc_period_start_block=is_calc_period_start_block, + instant_block_hash=instant_block_hash) From d04434ae0496ea4f438ff7ba622bd3edca600af6 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 19 Nov 2019 18:12:42 +0900 Subject: [PATCH 12/36] Update test_message_unpacker.py * RollbackResponse, QueryCalculateResultResponse and QueryCalculateStatusResponse are added to test --- iconservice/iiss/reward_calc/ipc/message.py | 2 +- .../iiss/reward_calc/ipc/message_unpacker.py | 3 +- tests/iiss/ipc/test_message_unpacker.py | 50 +++++++++++++++++-- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/iconservice/iiss/reward_calc/ipc/message.py b/iconservice/iiss/reward_calc/ipc/message.py index 07e8e7210..295c0062e 100644 --- a/iconservice/iiss/reward_calc/ipc/message.py +++ b/iconservice/iiss/reward_calc/ipc/message.py @@ -276,7 +276,7 @@ def from_list(items: list) -> 'QueryCalculateStatusResponse': payload: list = items[2] status: int = payload[0] - block_height: int = payload[2] + block_height: int = payload[1] return QueryCalculateStatusResponse(msg_id, status, block_height) diff --git a/iconservice/iiss/reward_calc/ipc/message_unpacker.py b/iconservice/iiss/reward_calc/ipc/message_unpacker.py index 2bc55446f..a511b8683 100644 --- a/iconservice/iiss/reward_calc/ipc/message_unpacker.py +++ b/iconservice/iiss/reward_calc/ipc/message_unpacker.py @@ -31,7 +31,8 @@ def __init__(self): MessageType.QUERY_CALCULATE_RESULT: QueryCalculateResultResponse, MessageType.INIT: InitResponse, MessageType.READY: ReadyNotification, - MessageType.CALCULATE_DONE: CalculateDoneNotification + MessageType.CALCULATE_DONE: CalculateDoneNotification, + MessageType.ROLLBACK: RollbackResponse, } def feed(self, data: bytes): diff --git a/tests/iiss/ipc/test_message_unpacker.py b/tests/iiss/ipc/test_message_unpacker.py index d1b17f59d..545d259c5 100644 --- a/tests/iiss/ipc/test_message_unpacker.py +++ b/tests/iiss/ipc/test_message_unpacker.py @@ -91,6 +91,24 @@ def test_iterator(self): MessageType.COMMIT_CLAIM, msg_id ), + ( + MessageType.QUERY_CALCULATE_STATUS, + msg_id, + ( + status, + block_height + ) + ), + ( + MessageType.QUERY_CALCULATE_RESULT, + msg_id, + ( + status, + block_height, + int_to_bytes(iscore), + state_hash + ) + ), ( MessageType.READY, msg_id, @@ -99,7 +117,6 @@ def test_iterator(self): block_height, block_hash ) - ), ( MessageType.CALCULATE_DONE, @@ -111,6 +128,15 @@ def test_iterator(self): state_hash ) ), + ( + MessageType.ROLLBACK, + msg_id, + ( + success, + block_height, + block_hash + ) + ), ( MessageType.INIT, msg_id, @@ -119,7 +145,7 @@ def test_iterator(self): block_height ) ) - ] + ] for message in messages: data: bytes = msgpack.packb(message) @@ -153,6 +179,17 @@ def test_iterator(self): commit_claim_response = next(it) self.assertIsInstance(commit_claim_response, CommitClaimResponse) + query_calculate_status = next(it) + self.assertIsInstance(query_calculate_status, QueryCalculateStatusResponse) + self.assertEqual(status, query_calculate_status.status) + self.assertEqual(block_height, query_calculate_status.block_height) + + query_calculate_result = next(it) + self.assertIsInstance(query_calculate_result, QueryCalculateResultResponse) + self.assertEqual(status, query_calculate_result.status) + self.assertEqual(block_height, query_calculate_result.block_height) + self.assertEqual(state_hash, query_calculate_result.state_hash) + ready_notification = next(it) self.assertIsInstance(ready_notification, ReadyNotification) self.assertEqual(version, ready_notification.version) @@ -165,6 +202,12 @@ def test_iterator(self): self.assertEqual(block_height, calculate_done_notification.block_height) self.assertEqual(state_hash, calculate_done_notification.state_hash) + rollback_response = next(it) + self.assertIsInstance(rollback_response, RollbackResponse) + self.assertTrue(rollback_response.success) + self.assertEqual(block_height, rollback_response.block_height) + self.assertEqual(block_hash, rollback_response.block_hash) + init_response = next(it) self.assertIsInstance(init_response, InitResponse) self.assertEqual(success, init_response.success) @@ -180,7 +223,8 @@ def test_iterator(self): expected = [ version_response, calculate_response, query_response, claim_response, commit_block_response, commit_claim_response, - ready_notification, calculate_done_notification + query_calculate_status, query_calculate_result, + ready_notification, calculate_done_notification, rollback_response ] for expected_response, response in zip(expected, self.unpacker): self.assertEqual(expected_response.MSG_TYPE, response.MSG_TYPE) From 45344cb8d589e097699d99634f94d9bb94fa3f0a Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 19 Nov 2019 18:13:58 +0900 Subject: [PATCH 13/36] Implement an unitest for BackupManager * Refactor BackupManager * Apply a new BackupManager to IconServiceEngine --- iconservice/database/backup_manager.py | 33 +++--- iconservice/icon_service_engine.py | 20 ++-- tests/database/test_backup_manager.py | 141 +++++++++++++++++++------ 3 files changed, 132 insertions(+), 62 deletions(-) diff --git a/iconservice/database/backup_manager.py b/iconservice/database/backup_manager.py index 6e7ce4143..54e7b147c 100644 --- a/iconservice/database/backup_manager.py +++ b/iconservice/database/backup_manager.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Optional from iconcommons import Logger + from .wal import WriteAheadLogWriter from ..database.db import KeyValueDatabase from ..icon_constant import ROLLBACK_LOG_TAG @@ -26,8 +27,6 @@ from .wal import IissWAL from ..base.block import Block from ..database.batch import BlockBatch - from ..precommit_data_manager import PrecommitData - TAG = ROLLBACK_LOG_TAG @@ -50,22 +49,16 @@ class BackupManager(object): """ - def __init__(self, state_db_root_path: str, rc_data_path: str, icx_db: 'KeyValueDatabase'): + def __init__(self, backup_root_path: str, rc_data_path: str): Logger.debug(tag=TAG, - msg=f"__init__() start: state_db_root_path={state_db_root_path}, " + msg=f"__init__() start: " + f"backup_root_path={backup_root_path}, " f"rc_data_path={rc_data_path}") self._rc_data_path = rc_data_path - self._root_path = os.path.join(state_db_root_path, "backup") - self._icx_db = icx_db - - Logger.info(tag=TAG, msg=f"backup_root_path={self._root_path}") - - try: - os.mkdir(self._root_path) - except FileExistsError: - pass + self._backup_root_path = backup_root_path + Logger.info(tag=TAG, msg=f"backup_root_path={self._backup_root_path}") Logger.debug(tag=TAG, msg="__init__() end") def _get_backup_file_path(self, block_height: int) -> str: @@ -77,11 +70,12 @@ def _get_backup_file_path(self, block_height: int) -> str: assert block_height >= 0 filename = get_backup_filename(block_height) - return os.path.join(self._root_path, filename) + return os.path.join(self._backup_root_path, filename) def run(self, - revision: int, + icx_db: 'KeyValueDatabase', rc_db: 'KeyValueDatabase', + revision: int, prev_block: 'Block', block_batch: 'BlockBatch', iiss_wal: 'IissWAL', @@ -89,8 +83,9 @@ def run(self, instant_block_hash: bytes): """Backup the previous block state - :param revision: + :param icx_db: :param rc_db: + :param revision: :param prev_block: the latest confirmed block height during commit :param block_batch: :param iiss_wal: @@ -113,7 +108,7 @@ def run(self, writer.write_state(WALBackupState.CALC_PERIOD_END_BLOCK.value) self._backup_rc_db(writer, rc_db, iiss_wal) - self._backup_state_db(writer, self._icx_db, block_batch) + self._backup_state_db(writer, icx_db, block_batch) writer.close() @@ -121,12 +116,12 @@ def run(self, def _clear_backup_files(self): try: - with os.scandir(self._root_path) as it: + with os.scandir(self._backup_root_path) as it: for entry in it: if entry.is_file() \ and entry.name.startswith("block-") \ and entry.name.endswith(".bak"): - path = os.path.join(self._root_path, entry.name) + path = os.path.join(self._backup_root_path, entry.name) self._remove_backup_file(path) except BaseException as e: Logger.info(tag=TAG, msg=str(e)) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index ee6f4bf18..c7f23cd70 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -131,10 +131,12 @@ def open(self, conf: 'IconConfig'): rc_data_path: str = os.path.join(state_db_root_path, IISS_DB) rc_socket_path: str = f"/tmp/iiss_{conf[ConfigKey.AMQP_KEY]}.sock" log_dir: str = os.path.dirname(conf[ConfigKey.LOG].get(ConfigKey.LOG_FILE_PATH, "./")) + backup_root_path: str = os.path.join(state_db_root_path, "backup") os.makedirs(score_root_path, exist_ok=True) os.makedirs(state_db_root_path, exist_ok=True) os.makedirs(rc_data_path, exist_ok=True) + os.makedirs(backup_root_path, exist_ok=True) # Share one context db with all SCORE ContextDatabaseFactory.open(state_db_root_path, ContextDatabaseFactory.Mode.SINGLE_DB) @@ -147,7 +149,7 @@ def open(self, conf: 'IconConfig'): self._deposit_handler = DepositHandler() self._icon_pre_validator = IconPreValidator() - self._backup_manager = BackupManager(state_db_root_path, rc_data_path, self._icx_context_db.key_value_db) + self._backup_manager = BackupManager(backup_root_path, rc_data_path) IconScoreClassLoader.init(score_root_path) IconScoreContext.score_root_path = score_root_path @@ -1830,13 +1832,15 @@ def _commit_after_iiss(self, wal_writer.flush() # Backup the previous block state - self._backup_manager.run(revision=context.revision, - rc_db=context.storage.rc.key_value_db, - prev_block=self._get_last_block(), - block_batch=precommit_data.block_batch, - iiss_wal=iiss_wal, - is_calc_period_start_block=is_calc_period_start_block, - instant_block_hash=instant_block_hash) + self._backup_manager.run( + icx_db=self._icx_context_db.key_value_db, + rc_db=context.storage.rc.key_value_db, + revision=context.revision, + prev_block=self._get_last_block(), + block_batch=precommit_data.block_batch, + iiss_wal=iiss_wal, + is_calc_period_start_block=is_calc_period_start_block, + instant_block_hash=instant_block_hash) # Write iiss_wal to rc_db standby_db_info: Optional['RewardCalcDBInfo'] = \ diff --git a/tests/database/test_backup_manager.py b/tests/database/test_backup_manager.py index 8d9e43882..f8d16f27e 100644 --- a/tests/database/test_backup_manager.py +++ b/tests/database/test_backup_manager.py @@ -13,29 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest -import shutil +import hashlib import os +import shutil +import unittest from collections import OrderedDict -import hashlib -from iconservice.icon_constant import Revision +from iconservice.base.block import Block from iconservice.database.backup_manager import BackupManager from iconservice.database.db import KeyValueDatabase -from iconservice.base.block import Block - +from iconservice.database.wal import WriteAheadLogReader, WALDBType +from iconservice.icon_constant import Revision -def _create_db(path) -> 'KeyValueDatabase': - db = KeyValueDatabase.from_path(path, create_if_missing=True) - for i in range(5): - postfix: bytes = i.to_bytes(1, "big", signed=False) - key: bytes = b"key" + postfix - value: bytes = b"value" + postfix +def _create_dummy_data(count: int) -> OrderedDict: + data = OrderedDict() - db.put(key, value) + for i in range(count): + key: bytes = f"key{i}".encode() + value: bytes = f"value{i}".encode() + data[key] = value - return db + return data class TestBackupManager(unittest.TestCase): @@ -43,27 +42,40 @@ def setUp(self) -> None: state_db_root_path = "./test_backup_manager_db" shutil.rmtree(state_db_root_path, ignore_errors=True) - os.mkdir(state_db_root_path) - state_db_path = os.path.join(state_db_root_path, "icon_dex") rc_data_path = os.path.join(state_db_root_path, "iiss") + backup_root_path = os.path.join(state_db_root_path, "backup") - icx_db = _create_db(state_db_path) + os.mkdir(state_db_root_path) + os.mkdir(rc_data_path) + os.mkdir(backup_root_path) + + org_state_db_data = _create_dummy_data(3) + icx_db = KeyValueDatabase.from_path(state_db_path, create_if_missing=True) + icx_db.write_batch(org_state_db_data.items()) + + org_rc_db_data = _create_dummy_data(0) + rc_db = KeyValueDatabase.from_path(rc_data_path) + rc_db.write_batch(org_rc_db_data.items()) self.backup_manager = BackupManager( - state_db_root_path=state_db_root_path, - rc_data_path=rc_data_path, - icx_db=icx_db + backup_root_path=backup_root_path, + rc_data_path=rc_data_path ) self._state_db_root_path = state_db_root_path - self._icx_db = icx_db - self.rc_db = KeyValueDatabase.from_path(rc_data_path) + self.backup_root_path = backup_root_path + + self.state_db = icx_db + self.org_state_db_data = org_state_db_data + + self.rc_db = rc_db + self.org_rc_db_data = org_rc_db_data def tearDown(self) -> None: - if self._icx_db: - self._icx_db.close() - self._icx_db = None + if self.state_db: + self.state_db.close() + self.state_db = None if self.rc_db: self.rc_db.close() @@ -78,7 +90,6 @@ def test_run(self): instant_block_hash: bytes = hashlib.sha3_256(b"instant_block_hash").digest() revision = Revision.DECENTRALIZATION.value - rc_db = self.rc_db last_block = Block( block_height=100, block_hash=block_hash, @@ -87,23 +98,83 @@ def test_run(self): cumulative_fee=0 ) block_batch = OrderedDict() - block_batch[b"key0"] = b"new value0" - block_batch[b"key1"] = None - block_batch[b"key2"] = b"value2" + block_batch[b"key0"] = b"new value0" # Update + block_batch[b"key1"] = None # Delete + block_batch[b"key2"] = b"value2" # Update with the same value + block_batch[b"key3"] = b"value3" # Add a new entry is_calc_period_start_block = False - rc_data = OrderedDict() - rc_data[b"key0"] = b"value0" + rc_batch = OrderedDict() + rc_batch[b"key0"] = b"hello" def iiss_wal(): - for key in rc_data: - yield key, rc_data[key] + for key in rc_batch: + yield key, rc_batch[key] - backup_manager.run(revision=revision, - rc_db=rc_db, + backup_manager.run(icx_db=self.state_db, + rc_db=self.rc_db, + revision=revision, prev_block=last_block, block_batch=block_batch, iiss_wal=iiss_wal(), is_calc_period_start_block=is_calc_period_start_block, instant_block_hash=instant_block_hash) + + self._commit_state_db(self.state_db, block_batch) + self._commit_rc_db(self.rc_db, rc_batch) + + self._rollback(last_block) + + @staticmethod + def _commit_state_db(db: 'KeyValueDatabase', block_batch: OrderedDict): + db.write_batch(block_batch.items()) + + for key in block_batch: + assert block_batch[key] == db.get(key) + + @staticmethod + def _commit_rc_db(db: 'KeyValueDatabase', rc_batch: OrderedDict): + db.write_batch(rc_batch.items()) + + count = 0 + for key in rc_batch: + assert rc_batch[key] == db.get(key) + count += 1 + + assert len(rc_batch) == count + + def _rollback(self, last_block: 'Block'): + backup_path = os.path.join(self.backup_root_path, f"block-{last_block.height}.bak") + + reader = WriteAheadLogReader() + reader.open(backup_path) + + self.rc_db.write_batch(reader.get_iterator(WALDBType.RC.value)) + self.state_db.write_batch(reader.get_iterator(WALDBType.STATE.value)) + + reader.close() + + self._check_if_rollback_is_done(self.rc_db, self.org_rc_db_data) + self._check_if_rollback_is_done(self.state_db, self.org_state_db_data) + + @staticmethod + def _rollback_and_check(reader: WriteAheadLogReader, db: 'KeyValueDatabase', prev_state: OrderedDict): + it = reader.get_iterator(WALDBType.STATE.value) + db.write_batch(it) + + i = 0 + for key, value in db.iterator(): + assert value == prev_state[key] + i += 1 + + assert i == len(prev_state) + + @staticmethod + def _check_if_rollback_is_done(db: 'KeyValueDatabase', prev_state: OrderedDict): + i = 0 + for key, value in db.iterator(): + assert value == prev_state[key] + i += 1 + + assert i == len(prev_state) From 17b3ee1ca9a21733eec9474f979be84f3455cbfc Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Wed, 20 Nov 2019 20:05:22 +0900 Subject: [PATCH 14/36] Refactor RollbackManager * Add unittest code to test_backup_manager.py --- iconservice/database/rollback_manager.py | 27 +++++------ iconservice/icon_service_engine.py | 6 +-- tests/database/test_backup_manager.py | 58 ++++++++++++++---------- 3 files changed, 52 insertions(+), 39 deletions(-) diff --git a/iconservice/database/rollback_manager.py b/iconservice/database/rollback_manager.py index 67801f79c..d95429ba5 100644 --- a/iconservice/database/rollback_manager.py +++ b/iconservice/database/rollback_manager.py @@ -35,20 +35,20 @@ class RollbackManager(object): - def __init__(self, icx_db: 'KeyValueDatabase', state_db_root_path: str, rc_data_path: str): - self._icx_db: 'KeyValueDatabase' = icx_db + def __init__(self, backup_root_path: str, rc_data_path: str): + self._backup_root_path = backup_root_path self._rc_data_path = rc_data_path - self._root_path = os.path.join(state_db_root_path, "backup") - def run(self, block_height: int = -1) -> Tuple[int, bool]: + def run(self, icx_db: 'KeyValueDatabase', block_height: int = -1) -> Tuple[int, bool]: """Rollback to the previous block state Called on self.open() + :param icx_db: state db :param block_height: the height of block to rollback to - :return: the height of block which is rollback to + :return: the height of block after rollback, is_calc_period_end_block """ - Logger.debug(tag=TAG, msg=f"rollback() start: BH={block_height}") + Logger.info(tag=TAG, msg=f"rollback() start: BH={block_height}") if block_height < 0: Logger.debug(tag=TAG, msg="rollback() end") @@ -70,7 +70,7 @@ def run(self, block_height: int = -1) -> Tuple[int, bool]: if reader.log_count == 2: self._rollback_rc_db(reader, is_calc_period_end_block) - self._rollback_state_db(reader) + self._rollback_state_db(reader, icx_db) block_height = reader.block.height except BaseException as e: @@ -81,7 +81,7 @@ def run(self, block_height: int = -1) -> Tuple[int, bool]: # Remove backup file after rollback is done self._remove_backup_file(path) - Logger.debug(tag=TAG, msg=f"rollback() end: return={block_height}") + Logger.info(tag=TAG, msg=f"rollback() end: return={block_height}, {is_calc_period_end_block}") return block_height, is_calc_period_end_block @@ -94,7 +94,7 @@ def _get_backup_file_path(self, block_height: int) -> str: assert block_height >= 0 filename = get_backup_filename(block_height) - return os.path.join(self._root_path, filename) + return os.path.join(self._backup_root_path, filename) def _rollback_rc_db(self, reader: 'WriteAheadLogReader', is_calc_period_end_block: bool): """Rollback the state of rc_db to the previous one @@ -141,17 +141,18 @@ def _rollback_rc_db_on_end_block(self, reader: 'WriteAheadLogReader'): self._remove_block_produce_info(current_rc_db_path, reader.block.height) - def _rollback_state_db(self, reader: 'WriteAheadLogReader'): - self._icx_db.write_batch(reader.get_iterator(WALDBType.STATE.value)) + @classmethod + def _rollback_state_db(cls, reader: 'WriteAheadLogReader', icx_db: 'KeyValueDatabase'): + icx_db.write_batch(reader.get_iterator(WALDBType.STATE.value)) def _clear_backup_files(self): try: - with os.scandir(self._root_path) as it: + with os.scandir(self._backup_root_path) as it: for entry in it: if entry.is_file() \ and entry.name.startswith("block-") \ and entry.name.endswith(".bak"): - path = os.path.join(self._root_path, entry.name) + path = os.path.join(self._backup_root_path, entry.name) self._remove_backup_file(path) except BaseException as e: Logger.info(tag=TAG, msg=str(e)) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index c7f23cd70..a29fb410e 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -99,6 +99,7 @@ def __init__(self): self._deposit_handler = None self._context_factory = None self._state_db_root_path: Optional[str] = None + self._backup_root_path: Optional[str] = None self._rc_data_path: Optional[str] = None self._wal_reader: Optional['WriteAheadLogReader'] = None self._backup_manager: Optional[BackupManager] = None @@ -2047,9 +2048,8 @@ def _rollback(self, context: 'IconScoreContext', block_height: int, block_hash: IconScoreContext.engine.iiss.rollback_reward_calculator(block_height, block_hash) # Rollback state_db and rc_data_db to those of a given block_height - rollback_manager = RollbackManager( - self._icx_context_db.key_value_db, self._state_db_root_path, self._rc_data_path) - rollback_manager.run(block_height) + rollback_manager = RollbackManager(self._backup_root_path, self._rc_data_path) + rollback_manager.run(self._icx_context_db.key_value_db, block_height) # Clear all iconscores and reload builtin scores only builtin_score_owner: 'Address' = Address.from_string(self._conf[ConfigKey.BUILTIN_SCORE_OWNER]) diff --git a/tests/database/test_backup_manager.py b/tests/database/test_backup_manager.py index f8d16f27e..f8e9f9d90 100644 --- a/tests/database/test_backup_manager.py +++ b/tests/database/test_backup_manager.py @@ -22,8 +22,10 @@ from iconservice.base.block import Block from iconservice.database.backup_manager import BackupManager from iconservice.database.db import KeyValueDatabase +from iconservice.database.rollback_manager import RollbackManager from iconservice.database.wal import WriteAheadLogReader, WALDBType from iconservice.icon_constant import Revision +from iconservice.iiss.reward_calc.storage import Storage as RewardCalcStorage def _create_dummy_data(count: int) -> OrderedDict: @@ -37,6 +39,10 @@ def _create_dummy_data(count: int) -> OrderedDict: return data +def _create_rc_db(rc_data_path: str) -> 'KeyValueDatabase': + return RewardCalcStorage.create_current_db(rc_data_path) + + class TestBackupManager(unittest.TestCase): def setUp(self) -> None: state_db_root_path = "./test_backup_manager_db" @@ -55,7 +61,7 @@ def setUp(self) -> None: icx_db.write_batch(org_state_db_data.items()) org_rc_db_data = _create_dummy_data(0) - rc_db = KeyValueDatabase.from_path(rc_data_path) + rc_db = _create_rc_db(rc_data_path) rc_db.write_batch(org_rc_db_data.items()) self.backup_manager = BackupManager( @@ -63,8 +69,14 @@ def setUp(self) -> None: rc_data_path=rc_data_path ) - self._state_db_root_path = state_db_root_path + self.rollback_manager = RollbackManager( + backup_root_path=backup_root_path, + rc_data_path=rc_data_path + ) + + self.state_db_root_path = state_db_root_path self.backup_root_path = backup_root_path + self.rc_data_path = rc_data_path self.state_db = icx_db self.org_state_db_data = org_state_db_data @@ -81,7 +93,7 @@ def tearDown(self) -> None: self.rc_db.close() self.rc_db = None - shutil.rmtree(self._state_db_root_path, ignore_errors=True) + shutil.rmtree(self.state_db_root_path, ignore_errors=True) def test_run(self): backup_manager = self.backup_manager @@ -108,23 +120,30 @@ def test_run(self): rc_batch = OrderedDict() rc_batch[b"key0"] = b"hello" - def iiss_wal(): - for key in rc_batch: - yield key, rc_batch[key] - backup_manager.run(icx_db=self.state_db, rc_db=self.rc_db, revision=revision, prev_block=last_block, block_batch=block_batch, - iiss_wal=iiss_wal(), + iiss_wal=rc_batch.items(), is_calc_period_start_block=is_calc_period_start_block, instant_block_hash=instant_block_hash) self._commit_state_db(self.state_db, block_batch) self._commit_rc_db(self.rc_db, rc_batch) - self._rollback(last_block) + self._check_if_rollback_is_done(self.rc_db, self.org_rc_db_data) + self._check_if_rollback_is_done(self.state_db, self.org_state_db_data) + + self._commit_state_db(self.state_db, block_batch) + self._commit_rc_db(self.rc_db, rc_batch) + self.rc_db.close() + self.rc_db = None + self._rollback_with_rollback_manager(last_block) + + self.rc_db = _create_rc_db(self.rc_data_path) + self._check_if_rollback_is_done(self.rc_db, self.org_rc_db_data) + self._check_if_rollback_is_done(self.state_db, self.org_state_db_data) @staticmethod def _commit_state_db(db: 'KeyValueDatabase', block_batch: OrderedDict): @@ -155,14 +174,8 @@ def _rollback(self, last_block: 'Block'): reader.close() - self._check_if_rollback_is_done(self.rc_db, self.org_rc_db_data) - self._check_if_rollback_is_done(self.state_db, self.org_state_db_data) - @staticmethod - def _rollback_and_check(reader: WriteAheadLogReader, db: 'KeyValueDatabase', prev_state: OrderedDict): - it = reader.get_iterator(WALDBType.STATE.value) - db.write_batch(it) - + def _check_if_rollback_is_done(db: 'KeyValueDatabase', prev_state: OrderedDict): i = 0 for key, value in db.iterator(): assert value == prev_state[key] @@ -170,11 +183,10 @@ def _rollback_and_check(reader: WriteAheadLogReader, db: 'KeyValueDatabase', pre assert i == len(prev_state) - @staticmethod - def _check_if_rollback_is_done(db: 'KeyValueDatabase', prev_state: OrderedDict): - i = 0 - for key, value in db.iterator(): - assert value == prev_state[key] - i += 1 + def _rollback_with_rollback_manager(self, last_block: 'Block'): + rollback_manager = self.rollback_manager - assert i == len(prev_state) + block_height, is_calc_end_block_height = \ + rollback_manager.run(self.state_db, last_block.height) + assert block_height == last_block.height + assert not is_calc_end_block_height From 294d1e935ebe2c14e1f1cebbe9e3cc9db598f0a7 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Thu, 21 Nov 2019 16:09:23 +0900 Subject: [PATCH 15/36] Fix minor bugs on rollback * Revert last_block * Fix unittest error in test_rollback.py --- iconservice/icon_service_engine.py | 5 +- .../iiss/decentralized/test_rollback.py | 159 ++++++++++++++++++ 2 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 tests/integrate_test/iiss/decentralized/test_rollback.py diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index a29fb410e..fe4255382 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -143,6 +143,7 @@ def open(self, conf: 'IconConfig'): ContextDatabaseFactory.open(state_db_root_path, ContextDatabaseFactory.Mode.SINGLE_DB) self._state_db_root_path = state_db_root_path self._rc_data_path = rc_data_path + self._backup_root_path = backup_root_path self._icx_context_db = ContextDatabaseFactory.create_by_name(ICON_DEX_DB_NAME) self._step_counter_factory = IconScoreStepCounterFactory() @@ -2029,7 +2030,7 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: def _is_rollback_needed(cls, last_block: 'Block', block_height: int, block_hash: bytes) -> bool: """Check if rollback is needed """ - if block_height == last_block.height + 1: + if block_height == last_block.height - 1: return True if block_height == last_block.height and block_hash == last_block.hash: return False @@ -2082,6 +2083,8 @@ def _rollback(self, context: 'IconScoreContext', block_height: int, block_hash: for engine in engines: engine.rollback(context, block_height, block_hash) + self._init_last_block_info(context) + def clear_context_stack(self): """Clear IconScoreContext stacks """ diff --git a/tests/integrate_test/iiss/decentralized/test_rollback.py b/tests/integrate_test/iiss/decentralized/test_rollback.py new file mode 100644 index 000000000..8da99f075 --- /dev/null +++ b/tests/integrate_test/iiss/decentralized/test_rollback.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""IconScoreEngine testcase +""" +from enum import Enum +from typing import TYPE_CHECKING, List, Dict +from unittest.mock import Mock + +from iconservice.icon_constant import ConfigKey, PRepGrade +from iconservice.icon_constant import PREP_MAIN_PREPS, PREP_MAIN_AND_SUB_PREPS +from iconservice.iconscore.icon_score_context import IconScoreContext +from iconservice.utils import icx_to_loop +from tests.integrate_test.iiss.test_iiss_base import TestIISSBase +from tests.integrate_test.test_integrate_base import EOAAccount + +if TYPE_CHECKING: + from iconservice.base.address import Address + from iconservice.base.block import Block + + +def _check_elected_prep_grades(preps: List[Dict[str, int]], + main_prep_count: int, + elected_prep_count: int): + for i, prep in enumerate(preps): + if i < main_prep_count: + assert prep["grade"] == PRepGrade.MAIN.value + elif i < elected_prep_count: + assert prep["grade"] == PRepGrade.SUB.value + else: + assert prep["grade"] == PRepGrade.CANDIDATE.value + + +def _get_main_preps(preps: List[Dict[str, 'Address']], main_prep_count: int) -> List['Address']: + main_preps: List['Address'] = [] + + for i, prep in enumerate(preps): + if i >= main_prep_count: + break + + main_preps.append(prep["address"]) + + return main_preps + + +class ProposalType(Enum): + TEXT = 0 + REVISION = 1 + MALICIOUS_SCORE = 2 + PREP_DISQUALIFICATION = 3 + STEP_PRICE = 4 + + +class TestRollback(TestIISSBase): + MAIN_PREP_COUNT = PREP_MAIN_PREPS + ELECTED_PREP_COUNT = PREP_MAIN_AND_SUB_PREPS + CALCULATE_PERIOD = MAIN_PREP_COUNT + TERM_PERIOD = MAIN_PREP_COUNT + BLOCK_VALIDATION_PENALTY_THRESHOLD = 10 + LOW_PRODUCTIVITY_PENALTY_THRESHOLD = 80 + PENALTY_GRACE_PERIOD = CALCULATE_PERIOD * 2 + BLOCK_VALIDATION_PENALTY_THRESHOLD + + def _make_init_config(self) -> dict: + return { + ConfigKey.SERVICE: { + ConfigKey.SERVICE_FEE: True + }, + ConfigKey.IISS_META_DATA: { + ConfigKey.UN_STAKE_LOCK_MIN: 10, + ConfigKey.UN_STAKE_LOCK_MAX: 20 + }, + ConfigKey.IISS_CALCULATE_PERIOD: self.CALCULATE_PERIOD, + ConfigKey.TERM_PERIOD: self.TERM_PERIOD, + ConfigKey.BLOCK_VALIDATION_PENALTY_THRESHOLD: self.BLOCK_VALIDATION_PENALTY_THRESHOLD, + ConfigKey.LOW_PRODUCTIVITY_PENALTY_THRESHOLD: self.LOW_PRODUCTIVITY_PENALTY_THRESHOLD, + ConfigKey.PREP_MAIN_PREPS: self.MAIN_PREP_COUNT, + ConfigKey.PREP_MAIN_AND_SUB_PREPS: self.ELECTED_PREP_COUNT, + ConfigKey.PENALTY_GRACE_PERIOD: self.PENALTY_GRACE_PERIOD + } + + def setUp(self): + super().setUp() + self.init_decentralized() + + # get main prep + response: dict = self.get_main_prep_list() + expected_preps: list = [] + for account in self._accounts[:PREP_MAIN_PREPS]: + expected_preps.append({ + 'address': account.address, + 'delegated': 0 + }) + expected_response: dict = { + "preps": expected_preps, + "totalDelegated": 0 + } + self.assertEqual(expected_response, response) + + def test_rollback_icx_transfer(self): + """ + scenario 1 + when it starts new preps on new term, normal case, while 100 block. + expected : + all new preps have maintained until 100 block because it already passed GRACE_PERIOD + """ + IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() + + # Inspect the current term + response = self.get_prep_term() + assert response["sequence"] == 2 + + prev_block: 'Block' = self.icon_service_engine._get_last_block() + + # Transfer 3000 icx to new 10 accounts + init_balance = icx_to_loop(3000) + accounts: List['EOAAccount'] = self.create_eoa_accounts(10) + self.distribute_icx(accounts=accounts, init_balance=init_balance) + + for account in accounts: + balance: int = self.get_balance(account.address) + assert balance == init_balance + + # Rollback the state to the previous block height + self.icon_service_engine.rollback(prev_block.height, prev_block.hash) + IconScoreContext.engine.iiss.rollback_reward_calculator.assert_called_with( + prev_block.height, prev_block.hash) + + # Check if the balances of accounts are reverted + for account in accounts: + balance: int = self.get_balance(account.address) + assert balance == 0 + + # Check the last block + self._check_if_last_block_is_reverted(prev_block) + + def test_rollback_score_deploy(self): + pass + + def _check_if_last_block_is_reverted(self, prev_block: 'Block'): + """After rollback, last_block should be the same as prev_block + + :param prev_block: + :return: + """ + last_block: 'Block' = self.icon_service_engine._get_last_block() + assert last_block == prev_block From 7781afa556a7ac7adc0d5767af056c1fa9face5f Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Fri, 22 Nov 2019 18:20:53 +0900 Subject: [PATCH 16/36] Improve unittest for rollback * Update create_eoa_account functions in TestIntegrateBase * Add a unittest for score deploy rollback * Add a unittest for score state rollback --- .../iiss/decentralized/test_rollback.py | 245 +++++++++++++++++- tests/integrate_test/test_integrate_base.py | 33 ++- 2 files changed, 265 insertions(+), 13 deletions(-) diff --git a/tests/integrate_test/iiss/decentralized/test_rollback.py b/tests/integrate_test/iiss/decentralized/test_rollback.py index 8da99f075..51ac688cb 100644 --- a/tests/integrate_test/iiss/decentralized/test_rollback.py +++ b/tests/integrate_test/iiss/decentralized/test_rollback.py @@ -20,16 +20,21 @@ from typing import TYPE_CHECKING, List, Dict from unittest.mock import Mock +import pytest + +from iconservice.base.address import Address +from iconservice.base.address import ZERO_SCORE_ADDRESS from iconservice.icon_constant import ConfigKey, PRepGrade from iconservice.icon_constant import PREP_MAIN_PREPS, PREP_MAIN_AND_SUB_PREPS +from iconservice.base.exception import ScoreNotFoundException from iconservice.iconscore.icon_score_context import IconScoreContext from iconservice.utils import icx_to_loop from tests.integrate_test.iiss.test_iiss_base import TestIISSBase from tests.integrate_test.test_integrate_base import EOAAccount if TYPE_CHECKING: - from iconservice.base.address import Address from iconservice.base.block import Block + from iconservice.iconscore.icon_score_result import TransactionResult def _check_elected_prep_grades(preps: List[Dict[str, int]], @@ -116,6 +121,7 @@ def test_rollback_icx_transfer(self): expected : all new preps have maintained until 100 block because it already passed GRACE_PERIOD """ + # Prevent icon_service_engine from sending RollbackRequest to rc IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() # Inspect the current term @@ -134,9 +140,7 @@ def test_rollback_icx_transfer(self): assert balance == init_balance # Rollback the state to the previous block height - self.icon_service_engine.rollback(prev_block.height, prev_block.hash) - IconScoreContext.engine.iiss.rollback_reward_calculator.assert_called_with( - prev_block.height, prev_block.hash) + self._rollback(prev_block) # Check if the balances of accounts are reverted for account in accounts: @@ -147,7 +151,233 @@ def test_rollback_icx_transfer(self): self._check_if_last_block_is_reverted(prev_block) def test_rollback_score_deploy(self): - pass + # Prevent icon_service_engine from sending RollbackRequest to rc + IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() + + init_balance = icx_to_loop(100) + deploy_step_limit = 2 * 10 ** 9 + sender_account: 'EOAAccount' = self.create_eoa_account() + sender_address: 'Address' = sender_account.address + + # Transfer 10 ICX to sender_account + self.distribute_icx([sender_account], init_balance=init_balance) + + # Save the balance of sender address + balance: int = self.get_balance(sender_address) + assert init_balance == balance + + # Save the previous block + prev_block: 'Block' = self.icon_service_engine._get_last_block() + + # Deploy a SCORE + tx: dict = self.create_deploy_score_tx(score_root="sample_deploy_scores", + score_name="install/sample_score", + from_=sender_address, + to_=ZERO_SCORE_ADDRESS, + step_limit=deploy_step_limit) + tx_results: List['TransactionResult'] = self.process_confirm_block(tx_list=[tx]) + + # Skip tx_result[0]. It is the result of a base transaction + tx_result: 'TransactionResult' = tx_results[1] + score_address: 'Address' = tx_result.score_address + assert isinstance(score_address, Address) + assert score_address.is_contract + + # Check if the score works well with a query request + response = self.query_score(from_=sender_address, to_=score_address, func_name="hello") + assert response == "Hello" + + # Check the balance is reduced + balance: int = self.get_balance(sender_address) + assert init_balance == balance + tx_result.step_price * tx_result.step_used + + # Rollback: Go back to the block where a score has not been deployed yet + self._rollback(prev_block) + + # Check if the score deployment is revoked successfully + with pytest.raises(ScoreNotFoundException): + self.query_score(from_=sender_address, to_=score_address, func_name="hello") + + # Check if the balance of sender address is revoked + balance: int = self.get_balance(sender_address) + assert init_balance == balance + + # Deploy the same SCORE again + tx: dict = self.create_deploy_score_tx(score_root="sample_deploy_scores", + score_name="install/sample_score", + from_=sender_address, + to_=ZERO_SCORE_ADDRESS, + step_limit=deploy_step_limit) + tx_results: List['TransactionResult'] = self.process_confirm_block(tx_list=[tx]) + + # Skip tx_result[0]. It is the result of a base transaction + tx_result: 'TransactionResult' = tx_results[1] + new_score_address = tx_result.score_address + assert isinstance(score_address, Address) + assert new_score_address.is_contract + assert score_address != new_score_address + + # Check if the score works well with a query request + response = self.query_score(from_=sender_address, to_=new_score_address, func_name="hello") + assert response == "Hello" + + def test_rollback_score_state(self): + # Prevent icon_service_engine from sending RollbackRequest to rc + IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() + + init_balance = icx_to_loop(100) + deploy_step_limit = 2 * 10 ** 9 + sender_account: 'EOAAccount' = self.create_eoa_account() + sender_address: 'Address' = sender_account.address + score_value = 1234 + deploy_params = {"value": hex(score_value)} + + # Transfer 10 ICX to sender_account + self.distribute_icx([sender_account], init_balance=init_balance) + + # Save the balance of sender address + balance: int = self.get_balance(sender_address) + assert init_balance == balance + + # Deploy a SCORE + tx: dict = self.create_deploy_score_tx(score_root="sample_deploy_scores", + score_name="install/sample_score", + from_=sender_address, + to_=ZERO_SCORE_ADDRESS, + deploy_params=deploy_params, + step_limit=deploy_step_limit) + tx_results: List['TransactionResult'] = self.process_confirm_block(tx_list=[tx]) + + # Skip tx_result[0]. It is the result of a base transaction + tx_result: 'TransactionResult' = tx_results[1] + score_address: 'Address' = tx_result.score_address + assert isinstance(score_address, Address) + assert score_address.is_contract + + # Check if the score works well with a query request + response = self.query_score(from_=sender_address, to_=score_address, func_name="get_value") + assert response == score_value + + # Check the balance is reduced + balance: int = self.get_balance(sender_address) + assert init_balance == balance + tx_result.step_price * tx_result.step_used + + # Save the previous block + prev_block: 'Block' = self.icon_service_engine._get_last_block() + + # Send a transaction to change the score state + old_balance = balance + tx_results: List['TransactionResult'] = self.score_call(from_=sender_address, + to_=score_address, + func_name="increase_value", + step_limit=10 ** 8, + expected_status=True) + + tx_result: 'TransactionResult' = tx_results[1] + assert tx_result.step_used > 0 + assert tx_result.step_price > 0 + assert tx_result.to == score_address + + balance: int = self.get_balance(sender_address) + assert old_balance == balance + tx_result.step_used * tx_result.step_price + + # Check if the score works well with a query request + response = self.query_score(from_=sender_address, to_=score_address, func_name="get_value") + assert response == score_value + 1 + + # Rollback: Go back to the block where a score has not been deployed yet + self._rollback(prev_block) + + # Check if the score state is reverted + response = self.query_score(from_=sender_address, to_=score_address, func_name="get_value") + assert response == score_value + + def test_rollback_register_prep(self): + # Prevent icon_service_engine from sending RollbackRequest to rc + IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() + + accounts: List['EOAAccount'] = self.create_eoa_accounts(1) + self.distribute_icx(accounts=accounts, init_balance=icx_to_loop(3000)) + + # Keep the previous states in order to compare with the rollback result + prev_get_preps: dict = self.get_prep_list() + prev_block: 'Block' = self.icon_service_engine._get_last_block() + + # Register a new P-Rep + transactions = [] + for i, account in enumerate(accounts): + # Register a P-Rep + tx = self.create_register_prep_tx(from_=account) + transactions.append(tx) + self.process_confirm_block_tx(transactions) + + # Check whether a registerPRep tx is done + current_get_preps: dict = self.get_prep_list() + assert current_get_preps["blockHeight"] == prev_block.height + 1 + assert len(current_get_preps["preps"]) == len(prev_get_preps["preps"]) + 1 + + # Rollback + self._rollback(prev_block) + + current_get_preps: dict = self.get_prep_list() + assert current_get_preps == prev_get_preps + + self._check_if_last_block_is_reverted(prev_block) + + def test_rollback_set_delegation(self): + # Prevent icon_service_engine from sending RollbackRequest to rc + IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() + + accounts: List['EOAAccount'] = self.create_eoa_accounts(1) + self.distribute_icx(accounts=accounts, init_balance=icx_to_loop(3000)) + user_account = accounts[0] + + # Keep the previous states in order to compare with the rollback result + prev_get_preps: dict = self.get_prep_list() + prev_block: 'Block' = self.icon_service_engine._get_last_block() + + # Move 22th P-Rep up to 1st with setDelegation + delegating: int = icx_to_loop(1) + transactions = [] + for i, account in enumerate(accounts): + # Stake 100 icx + tx = self.create_set_stake_tx(from_=user_account, value=icx_to_loop(100)) + transactions.append(tx) + + # Delegate 1 icx to itself + tx = self.create_set_delegation_tx( + from_=account, + origin_delegations=[ + (self._accounts[PREP_MAIN_PREPS - 1], delegating) + ] + ) + transactions.append(tx) + self.process_confirm_block_tx(transactions) + + # Check whether a setDelegation tx is done + current_get_preps: dict = self.get_prep_list() + assert current_get_preps["blockHeight"] == prev_block.height + 1 + + prev_prep_info: dict = prev_get_preps["preps"][PREP_MAIN_PREPS - 1] + current_prep_info: dict = current_get_preps["preps"][0] + for field in prev_prep_info: + if field == "delegated": + assert prev_prep_info[field] == current_prep_info[field] - delegating + else: + assert prev_prep_info[field] == current_prep_info[field] + + # Rollback + self._rollback(prev_block) + + current_get_preps: dict = self.get_prep_list() + assert current_get_preps == prev_get_preps + + self._check_if_last_block_is_reverted(prev_block) + + def _rollback(self, block: 'Block'): + super().rollback(block.height, block.hash) + self._check_if_rollback_reward_calculator_is_called(block) + self._check_if_last_block_is_reverted(block) def _check_if_last_block_is_reverted(self, prev_block: 'Block'): """After rollback, last_block should be the same as prev_block @@ -157,3 +387,8 @@ def _check_if_last_block_is_reverted(self, prev_block: 'Block'): """ last_block: 'Block' = self.icon_service_engine._get_last_block() assert last_block == prev_block + + @staticmethod + def _check_if_rollback_reward_calculator_is_called(block: 'Block'): + IconScoreContext.engine.iiss.rollback_reward_calculator.assert_called_with( + block.height, block.hash) diff --git a/tests/integrate_test/test_integrate_base.py b/tests/integrate_test/test_integrate_base.py index 5958b1467..0e217dcbd 100644 --- a/tests/integrate_test/test_integrate_base.py +++ b/tests/integrate_test/test_integrate_base.py @@ -108,6 +108,9 @@ def mock_calculate(self, _path, _block_height): response = CalculateDoneNotification(0, True, end_block_height_of_calc - calc_period, 0, b'mocked_response') self._calculate_done_callback(response) + def _calculate_done_callback(self, response: 'CalculateDoneNotification'): + pass + @classmethod def _mock_ipc(cls, mock_calculate: callable = mock_calculate): RewardCalcProxy.open = Mock() @@ -291,7 +294,21 @@ def _write_precommit_state(self, block: 'Block') -> None: self._prev_block_hash = block.hash def _remove_precommit_state(self, block: 'Block') -> None: - self.icon_service_engine.rollback(block.height, block.hash) + """Revoke to commit the precommit data to db + + """ + self.icon_service_engine.remove_precommit_state(block.height, block.hash) + + def rollback(self, block_height: int = -1, block_hash: Optional[bytes] = None): + """Rollback the current state to the old one indicated by a given block + + :param block_height: the final block height after rollback + :param block_hash: the final block hash after rollback + """ + self.icon_service_engine.rollback(block_height, block_hash) + + self._block_height = block_height + self._prev_block_hash = block_hash def _query(self, request: dict, method: str = 'icx_call') -> Any: response = self.icon_service_engine.query(method, request) @@ -877,13 +894,13 @@ def get_step_price(self) -> int: } return self._query(query_request) - @staticmethod - def create_eoa_accounts(count: int) -> List['EOAAccount']: - accounts: list = [] - wallets: List['KeyWallet'] = [KeyWallet.create() for _ in range(count)] - for wallet in wallets: - accounts.append(EOAAccount(wallet)) - return accounts + @classmethod + def create_eoa_account(cls) -> 'EOAAccount': + return EOAAccount(KeyWallet.create()) + + @classmethod + def create_eoa_accounts(cls, count: int) -> List['EOAAccount']: + return [cls.create_eoa_account() for _ in range(count)] class EOAAccount: From 2c0f9bb23fee2fbbb424def4ba70f0e90eaffdc8 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Wed, 4 Dec 2019 20:17:08 +0900 Subject: [PATCH 17/36] Fix a minor bug in IconInnerService.rollback() --- iconservice/icon_inner_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iconservice/icon_inner_service.py b/iconservice/icon_inner_service.py index 554506c73..4bc43f6c3 100644 --- a/iconservice/icon_inner_service.py +++ b/iconservice/icon_inner_service.py @@ -367,6 +367,8 @@ async def rollback(self, request: dict): Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"rollback() end: {response}") + return response + def _rollback(self, request: dict) -> dict: Logger.info(tag=ICON_INNER_LOG_TAG, msg=f"_rollback() start: {request}") From 8e65485430e9cb67378dd1d61f5c6c9a419d1b1c Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 10 Dec 2019 16:12:50 +0900 Subject: [PATCH 18/36] Implement BackupCleaner * Move rollback-related files to iconservice.rollback package * Apply BackupCleaner to IconServiceEngine * Add a unit test for BackupCleaner --- iconservice/icon_config.py | 10 +- iconservice/icon_constant.py | 5 + iconservice/icon_service_engine.py | 16 ++- iconservice/rollback/__init__.py | 26 +++++ iconservice/rollback/backup_cleaner.py | 68 ++++++++++++ .../{database => rollback}/backup_manager.py | 45 ++------ .../rollback_manager.py | 14 +-- tests/rollback/__init__.py | 0 tests/rollback/test_backup_cleaner.py | 104 ++++++++++++++++++ .../test_backup_manager.py | 4 +- 10 files changed, 239 insertions(+), 53 deletions(-) create mode 100644 iconservice/rollback/__init__.py create mode 100644 iconservice/rollback/backup_cleaner.py rename iconservice/{database => rollback}/backup_manager.py (75%) rename iconservice/{database => rollback}/rollback_manager.py (93%) create mode 100644 tests/rollback/__init__.py create mode 100644 tests/rollback/test_backup_cleaner.py rename tests/{database => rollback}/test_backup_manager.py (98%) diff --git a/iconservice/icon_config.py b/iconservice/icon_config.py index 207485d84..1ca60a379 100644 --- a/iconservice/icon_config.py +++ b/iconservice/icon_config.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .icon_constant import ConfigKey, ICX_IN_LOOP, TERM_PERIOD, IISS_DAY_BLOCK, PREP_MAIN_PREPS, \ - PREP_MAIN_AND_SUB_PREPS, PENALTY_GRACE_PERIOD, LOW_PRODUCTIVITY_PENALTY_THRESHOLD, BLOCK_VALIDATION_PENALTY_THRESHOLD +from .icon_constant import ( + ConfigKey, ICX_IN_LOOP, TERM_PERIOD, IISS_DAY_BLOCK, PREP_MAIN_PREPS, + PREP_MAIN_AND_SUB_PREPS, PENALTY_GRACE_PERIOD, LOW_PRODUCTIVITY_PENALTY_THRESHOLD, + BLOCK_VALIDATION_PENALTY_THRESHOLD, BACKUP_FILES +) default_icon_config = { "log": { @@ -57,5 +60,6 @@ ConfigKey.LOW_PRODUCTIVITY_PENALTY_THRESHOLD: LOW_PRODUCTIVITY_PENALTY_THRESHOLD, ConfigKey.BLOCK_VALIDATION_PENALTY_THRESHOLD: BLOCK_VALIDATION_PENALTY_THRESHOLD, ConfigKey.STEP_TRACE_FLAG: False, - ConfigKey.PRECOMMIT_DATA_LOG_FLAG: False + ConfigKey.PRECOMMIT_DATA_LOG_FLAG: False, + ConfigKey.BACKUP_FILES: BACKUP_FILES } diff --git a/iconservice/icon_constant.py b/iconservice/icon_constant.py index 977189e55..196d96ae4 100644 --- a/iconservice/icon_constant.py +++ b/iconservice/icon_constant.py @@ -183,6 +183,9 @@ class ConfigKey: LOW_PRODUCTIVITY_PENALTY_THRESHOLD = "lowProductivityPenaltyThreshold" BLOCK_VALIDATION_PENALTY_THRESHOLD = "blockValidationPenaltyThreshold" + # The maximum number of backup files for rollback + BACKUP_FILES = "backupFiles" + class EnableThreadFlag(IntFlag): INVOKE = 1 @@ -301,6 +304,8 @@ class DeployState(IntEnum): PREP_PENALTY_SIGNATURE = "PenaltyImposed(Address,int,int)" +BACKUP_FILES = 10 + class RCStatus(IntEnum): NOT_READY = 0 diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index fe4255382..8f6a9a729 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -19,8 +19,9 @@ from iconcommons.logger import Logger -from iconservice.database.backup_manager import BackupManager -from iconservice.database.rollback_manager import RollbackManager +from iconservice.rollback.backup_cleaner import BackupCleaner +from iconservice.rollback.backup_manager import BackupManager +from iconservice.rollback.rollback_manager import RollbackManager from .base.address import Address, generate_score_address, generate_score_address_for_tbears from .base.address import ZERO_SCORE_ADDRESS, GOVERNANCE_SCORE_ADDRESS from .base.block import Block, EMPTY_BLOCK @@ -41,7 +42,8 @@ ICON_DEX_DB_NAME, ICON_SERVICE_LOG_TAG, IconServiceFlag, ConfigKey, IISS_METHOD_TABLE, PREP_METHOD_TABLE, NEW_METHOD_TABLE, Revision, BASE_TRANSACTION_INDEX, IISS_DB, IISS_INITIAL_IREP, DEBUG_METHOD_TABLE, PREP_MAIN_PREPS, PREP_MAIN_AND_SUB_PREPS, - ISCORE_EXCHANGE_RATE, STEP_LOG_TAG, TERM_PERIOD, BlockVoteStatus, WAL_LOG_TAG, ROLLBACK_LOG_TAG) + ISCORE_EXCHANGE_RATE, STEP_LOG_TAG, TERM_PERIOD, BlockVoteStatus, WAL_LOG_TAG, ROLLBACK_LOG_TAG +) from .iconscore.icon_pre_validator import IconPreValidator from .iconscore.icon_score_class_loader import IconScoreClassLoader from .iconscore.icon_score_context import IconScoreContext, IconScoreFuncType, ContextContainer, IconScoreContextFactory @@ -103,6 +105,7 @@ def __init__(self): self._rc_data_path: Optional[str] = None self._wal_reader: Optional['WriteAheadLogReader'] = None self._backup_manager: Optional[BackupManager] = None + self._backup_cleaner: Optional[BackupCleaner] = None self._conf: Optional[Dict[str, Union[str, int]]] = None # JSON-RPC handlers @@ -152,6 +155,7 @@ def open(self, conf: 'IconConfig'): self._deposit_handler = DepositHandler() self._icon_pre_validator = IconPreValidator() self._backup_manager = BackupManager(backup_root_path, rc_data_path) + self._backup_cleaner = BackupCleaner(backup_root_path, conf[ConfigKey.BACKUP_FILES]) IconScoreClassLoader.init(score_root_path) IconScoreContext.score_root_path = score_root_path @@ -174,6 +178,9 @@ def open(self, conf: 'IconConfig'): context = IconScoreContext(IconScoreContextType.DIRECT) self._init_last_block_info(context) + # Clean up stale backup files + self._backup_cleaner.run(context.block.height) + # set revision (if governance SCORE does not exist, remain revision to default). try: self._set_revision_to_context(context) @@ -1844,6 +1851,9 @@ def _commit_after_iiss(self, is_calc_period_start_block=is_calc_period_start_block, instant_block_hash=instant_block_hash) + # Clean up stale backup files + self._backup_cleaner.run(context.block.height) + # Write iiss_wal to rc_db standby_db_info: Optional['RewardCalcDBInfo'] = \ self._process_iiss_commit(context, precommit_data, iiss_wal, is_calc_period_start_block) diff --git a/iconservice/rollback/__init__.py b/iconservice/rollback/__init__.py new file mode 100644 index 000000000..aaa7b26b0 --- /dev/null +++ b/iconservice/rollback/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +__all__ = "get_backup_filename" + + +def get_backup_filename(block_height: int) -> str: + """ + + :param block_height: the height of the block where we want to go back + :return: + """ + return f"block-{block_height}.bak" diff --git a/iconservice/rollback/backup_cleaner.py b/iconservice/rollback/backup_cleaner.py new file mode 100644 index 000000000..cc7959546 --- /dev/null +++ b/iconservice/rollback/backup_cleaner.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = "BackupCleaner" + +import os + +from iconcommons.logger import Logger + +from . import get_backup_filename +from ..icon_constant import BACKUP_LOG_TAG, BACKUP_FILES + +_TAG = BACKUP_LOG_TAG + + +class BackupCleaner(object): + """Remove old backup files to rollback + + """ + + def __init__(self, backup_root_path: str, backup_files: int): + """ + + :param backup_root_path: the directory where backup files are placed + :param backup_files: the maximum backup files to keep + """ + self._backup_root_path = backup_root_path + self._backup_files = backup_files if backup_files >= 0 else BACKUP_FILES + + def run(self, current_block_height: int) -> int: + """Clean up old backup files + + :param: current_block_height + :param: func: function to remove a file with path + :return: the number of removed files + """ + Logger.debug(tag=_TAG, msg=f"run() start: current_block_height={current_block_height}") + + ret = 0 + start = current_block_height - self._backup_files - 1 + + try: + for block_height in range(start, -1, -1): + filename: str = get_backup_filename(block_height) + path: str = os.path.join(self._backup_root_path, filename) + os.remove(path) + ret += 1 + except FileNotFoundError: + pass + except BaseException as e: + Logger.debug(tag=_TAG, msg=str(e)) + + Logger.info(tag=_TAG, msg=f"Clean up old backup files: start={start} count={ret}") + Logger.debug(tag=_TAG, msg="run() end") + + return ret diff --git a/iconservice/database/backup_manager.py b/iconservice/rollback/backup_manager.py similarity index 75% rename from iconservice/database/backup_manager.py rename to iconservice/rollback/backup_manager.py index 54e7b147c..e3a309478 100644 --- a/iconservice/database/backup_manager.py +++ b/iconservice/rollback/backup_manager.py @@ -19,27 +19,19 @@ from iconcommons import Logger -from .wal import WriteAheadLogWriter -from ..database.db import KeyValueDatabase -from ..icon_constant import ROLLBACK_LOG_TAG +from iconservice.database.db import KeyValueDatabase +from iconservice.database.wal import WriteAheadLogWriter +from iconservice.icon_constant import ROLLBACK_LOG_TAG +from iconservice.rollback import get_backup_filename if TYPE_CHECKING: - from .wal import IissWAL - from ..base.block import Block - from ..database.batch import BlockBatch + from iconservice.database.wal import IissWAL + from iconservice.base.block import Block + from iconservice.database.batch import BlockBatch TAG = ROLLBACK_LOG_TAG -def get_backup_filename(block_height: int) -> str: - """ - - :param block_height: the height of the block where we want to go back - :return: - """ - return f"block-{block_height}.bak" - - class WALBackupState(Flag): CALC_PERIOD_END_BLOCK = 1 @@ -98,8 +90,6 @@ def run(self, path: str = self._get_backup_file_path(prev_block.height) Logger.info(tag=TAG, msg=f"backup_file_path={path}") - self._clear_backup_files() - writer = WriteAheadLogWriter( revision, max_log_count=2, block=prev_block, instant_block_hash=instant_block_hash) writer.open(path) @@ -114,27 +104,6 @@ def run(self, Logger.debug(tag=TAG, msg="backup() end") - def _clear_backup_files(self): - try: - with os.scandir(self._backup_root_path) as it: - for entry in it: - if entry.is_file() \ - and entry.name.startswith("block-") \ - and entry.name.endswith(".bak"): - path = os.path.join(self._backup_root_path, entry.name) - self._remove_backup_file(path) - except BaseException as e: - Logger.info(tag=TAG, msg=str(e)) - - @classmethod - def _remove_backup_file(cls, path: str): - try: - os.remove(path) - except FileNotFoundError: - pass - except BaseException as e: - Logger.debug(tag=TAG, msg=str(e)) - @classmethod def _backup_rc_db(cls, writer: 'WriteAheadLogWriter', db: 'KeyValueDatabase', iiss_wal: 'IissWAL'): def get_rc_db_generator(): diff --git a/iconservice/database/rollback_manager.py b/iconservice/rollback/rollback_manager.py similarity index 93% rename from iconservice/database/rollback_manager.py rename to iconservice/rollback/rollback_manager.py index d95429ba5..2baa0dd96 100644 --- a/iconservice/database/rollback_manager.py +++ b/iconservice/rollback/rollback_manager.py @@ -20,15 +20,15 @@ from iconcommons.logger import Logger from .backup_manager import WALBackupState, get_backup_filename -from .db import KeyValueDatabase -from .wal import WriteAheadLogReader, WALDBType -from ..base.exception import DatabaseException -from ..icon_constant import ROLLBACK_LOG_TAG -from ..iiss.reward_calc import RewardCalcStorage -from ..iiss.reward_calc.msg_data import make_block_produce_info_key +from iconservice.database.db import KeyValueDatabase +from iconservice.database.wal import WriteAheadLogReader, WALDBType +from iconservice.base.exception import DatabaseException +from iconservice.icon_constant import ROLLBACK_LOG_TAG +from iconservice.iiss.reward_calc import RewardCalcStorage +from iconservice.iiss.reward_calc.msg_data import make_block_produce_info_key if TYPE_CHECKING: - from .db import KeyValueDatabase + from iconservice.database.db import KeyValueDatabase TAG = ROLLBACK_LOG_TAG diff --git a/tests/rollback/__init__.py b/tests/rollback/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/rollback/test_backup_cleaner.py b/tests/rollback/test_backup_cleaner.py new file mode 100644 index 000000000..7e919712e --- /dev/null +++ b/tests/rollback/test_backup_cleaner.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import os +import shutil + +from iconservice.rollback.backup_cleaner import BackupCleaner +from iconservice.rollback import get_backup_filename +from iconservice.icon_constant import BACKUP_FILES + + +def _create_dummy_backup_files(backup_root_path: str, current_block_height: int, backup_files: int): + for i in range(backup_files): + block_height = current_block_height - i - 1 + if block_height < 0: + break + + filename = get_backup_filename(block_height) + path: str = os.path.join(backup_root_path, filename) + open(path, "w").close() + + assert os.path.isfile(path) + + +def _check_if_backup_files_exists(backup_root_path: str, start_block_height, count, expected: bool): + for i in range(count): + block_height = start_block_height + i + filename = get_backup_filename(block_height) + path = os.path.join(backup_root_path, filename) + + assert os.path.exists(path) == expected + + +class TestBackupCleaner(unittest.TestCase): + def setUp(self) -> None: + backup_files = 10 + backup_root_path = os.path.join(os.path.dirname(__file__), "backup") + os.mkdir(backup_root_path) + + backup_cleaner = BackupCleaner(backup_root_path, backup_files) + + self.backup_files = backup_files + self.backup_root_path = backup_root_path + self.backup_cleaner = backup_cleaner + + def tearDown(self) -> None: + shutil.rmtree(self.backup_root_path) + + def test__init__(self): + backup_cleaner = BackupCleaner("./haha", backup_files=-10) + assert backup_cleaner._backup_files >= 0 + assert backup_cleaner._backup_files == BACKUP_FILES + + def test_run_with_too_many_backup_files(self): + current_block_height = 101 + dummy_backup_files = 20 + backup_files = 10 + backup_root_path: str = self.backup_root_path + backup_cleaner = BackupCleaner(backup_root_path, backup_files) + + # Create 20 dummy backup files: block-81.bak ... block-100.back + _create_dummy_backup_files(backup_root_path, current_block_height, dummy_backup_files) + + # Remove block-81.bak ~ block-90.bak + backup_cleaner.run(current_block_height) + + # Check if too old backup files are removed + _check_if_backup_files_exists(backup_root_path, 81, 10, expected=False) + + # Check if the latest backup files exist (block-91.bak ~ block-100.bak) + _check_if_backup_files_exists(backup_root_path, 91, 10, expected=True) + + def test_run_with_too_short_backup_files(self): + current_block_height = 101 + dummy_backup_files = 5 + backup_files = 10 + backup_root_path: str = self.backup_root_path + backup_cleaner = BackupCleaner(backup_root_path, backup_files) + + # Create 5 dummy backup files: block-96.bak ... block-100.back + _create_dummy_backup_files(backup_root_path, current_block_height, dummy_backup_files) + + # No backup file will be removed + backup_cleaner.run(current_block_height) + + # Check if the latest backup files exist (block-96.bak ~ block-100.bak) + _check_if_backup_files_exists(backup_root_path, 96, 5, expected=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/database/test_backup_manager.py b/tests/rollback/test_backup_manager.py similarity index 98% rename from tests/database/test_backup_manager.py rename to tests/rollback/test_backup_manager.py index f8e9f9d90..1ee7e202a 100644 --- a/tests/database/test_backup_manager.py +++ b/tests/rollback/test_backup_manager.py @@ -20,9 +20,9 @@ from collections import OrderedDict from iconservice.base.block import Block -from iconservice.database.backup_manager import BackupManager +from iconservice.rollback.backup_manager import BackupManager from iconservice.database.db import KeyValueDatabase -from iconservice.database.rollback_manager import RollbackManager +from iconservice.rollback.rollback_manager import RollbackManager from iconservice.database.wal import WriteAheadLogReader, WALDBType from iconservice.icon_constant import Revision from iconservice.iiss.reward_calc.storage import Storage as RewardCalcStorage From 3cd060ff9d6b323afc3116e5fe528bfa6674af45 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 10 Dec 2019 17:56:28 +0900 Subject: [PATCH 19/36] Minor update * Update logging of RollbackResponse * Fix minor bug in IISSEngine.rollback_reward_calculator() --- iconservice/iiss/engine.py | 12 ++++++------ iconservice/iiss/reward_calc/ipc/message.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/iconservice/iiss/engine.py b/iconservice/iiss/engine.py index 4b1ac8c9b..4b6d85d4a 100644 --- a/iconservice/iiss/engine.py +++ b/iconservice/iiss/engine.py @@ -915,15 +915,15 @@ def rollback_reward_calculator(self, block_height: int, block_hash: bytes): msg=f"rollback_reward_calculator() start: " f"height={block_height} hash={bytes_to_hex(block_hash)}") - _status, _height, _hash = self._reward_calc_proxy.rollback(block_height, block_hash) + _success, _height, _hash = self._reward_calc_proxy.rollback(block_height, block_hash) Logger.info(tag=ROLLBACK_LOG_TAG, msg=f"RewardCalculator response: " - f"status={_status} height={_height} hash={bytes_to_hex(_hash)}") + f"success={_success} height={_height} hash={bytes_to_hex(_hash)}") - if _status and _height == block_height and _hash == block_hash: - Logger.info(tag=ROLLBACK_LOG_TAG, - msg=f"rollback_reward_calculator() end: " - f"height={block_height} hash={bytes_to_hex(block_hash)}") + # Reward calculator rollback succeeded + if _success and _height == block_height and _hash == block_hash: + Logger.info(tag=ROLLBACK_LOG_TAG, msg=f"rollback_reward_calculator() end") + return raise InternalServiceErrorException("Failed to rollback RewardCalculator") diff --git a/iconservice/iiss/reward_calc/ipc/message.py b/iconservice/iiss/reward_calc/ipc/message.py index 295c0062e..63aec1a2d 100644 --- a/iconservice/iiss/reward_calc/ipc/message.py +++ b/iconservice/iiss/reward_calc/ipc/message.py @@ -513,7 +513,7 @@ def __init__(self, msg_id: int, success: bool, block_height: int, block_hash: by self.block_hash: bytes = block_hash def __str__(self): - return f"ROLLBACK({self.msg_id}, {self.block_height}, {bytes_to_hex(self.block_hash)})" + return f"ROLLBACK({self.msg_id}, {self.success}, {self.block_height}, {bytes_to_hex(self.block_hash)})" @staticmethod def from_list(items: list) -> 'RollbackResponse': From 36c0ab192b0f081d6fbafb8ae6a8676794e8d4eb Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Wed, 11 Dec 2019 16:06:51 +0900 Subject: [PATCH 20/36] Add more detail logs to RollbackManager --- iconservice/rollback/rollback_manager.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/iconservice/rollback/rollback_manager.py b/iconservice/rollback/rollback_manager.py index 2baa0dd96..c9365a9f3 100644 --- a/iconservice/rollback/rollback_manager.py +++ b/iconservice/rollback/rollback_manager.py @@ -78,7 +78,7 @@ def run(self, icx_db: 'KeyValueDatabase', block_height: int = -1) -> Tuple[int, finally: reader.close() - # Remove backup file after rollback is done + # Remove the backup file after rollback is done self._remove_backup_file(path) Logger.info(tag=TAG, msg=f"rollback() end: return={block_height}, {is_calc_period_end_block}") @@ -103,6 +103,8 @@ def _rollback_rc_db(self, reader: 'WriteAheadLogReader', is_calc_period_end_bloc :param is_calc_period_end_block: :return: """ + Logger.debug(tag=TAG, msg=f"_rollback_rc_db() start: is_end_block={is_calc_period_end_block}") + if is_calc_period_end_block: Logger.info(tag=TAG, msg=f"BH-{reader.block.height} is a calc period end block") self._rollback_rc_db_on_end_block(reader) @@ -111,12 +113,15 @@ def _rollback_rc_db(self, reader: 'WriteAheadLogReader', is_calc_period_end_bloc db.write_batch(reader.get_iterator(WALDBType.RC.value)) db.close() + Logger.debug(tag=TAG, msg=f"_rollback_rc_db() end") + def _rollback_rc_db_on_end_block(self, reader: 'WriteAheadLogReader'): """ :param reader: :return: """ + Logger.debug(tag=TAG, msg=f"_rollback_rc_db_on_end_block() start") current_rc_db_path, standby_rc_db_path, iiss_rc_db_path = \ RewardCalcStorage.scan_rc_db(self._rc_data_path) @@ -130,17 +135,25 @@ def _rollback_rc_db_on_end_block(self, reader: 'WriteAheadLogReader'): if iiss_rc_db_exists: # Remove the next calc_period current_rc_db and rename iiss_rc_db to current_rc_db shutil.rmtree(current_rc_db_path) - os.rename(iiss_rc_db_path, current_rc_db_path) + self._rename_rc_db(iiss_rc_db_path, current_rc_db_path) else: if iiss_rc_db_exists: # iiss_rc_db -> current_rc_db - os.rename(iiss_rc_db_path, current_rc_db_path) + self._rename_rc_db(iiss_rc_db_path, current_rc_db_path) else: # If both current_rc_db and iiss_rc_db do not exist, raise error raise DatabaseException(f"RC DB not found") self._remove_block_produce_info(current_rc_db_path, reader.block.height) + Logger.debug(tag=TAG, msg=f"_rollback_rc_db_on_end_block() end") + + @classmethod + def _rename_rc_db(cls, src_path: str, dst_path: str): + Logger.info(tag=TAG, msg=f"_rename_rc_db() start: src={src_path} dst={dst_path}") + os.rename(src_path, dst_path) + Logger.info(tag=TAG, msg=f"_rename_rc_db() end") + @classmethod def _rollback_state_db(cls, reader: 'WriteAheadLogReader', icx_db: 'KeyValueDatabase'): icx_db.write_batch(reader.get_iterator(WALDBType.STATE.value)) @@ -174,8 +187,13 @@ def _remove_block_produce_info(cls, db_path: str, block_height: int): :param block_height: :return: """ + Logger.debug(tag=TAG, + msg=f"_remove_block_produce_info() start: db_path={db_path} block_height={block_height}") + key: bytes = make_block_produce_info_key(block_height) db = KeyValueDatabase.from_path(db_path, create_if_missing=False) db.delete(key) db.close() + + Logger.debug(tag=TAG, msg="_remove_block_produce_info() end") From 1e7b5bb9f0d7b9a2afa9badb78d3b544d4c304eb Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Wed, 11 Dec 2019 20:48:56 +0900 Subject: [PATCH 21/36] Fix a crash on RCStorage.get_total_elected_prep_delegated_snapshot() * The exception that there is no preps data at the end of prevote term is handled. --- iconservice/iiss/reward_calc/storage.py | 10 ++++------ iconservice/prep/data/term.py | 3 +++ iconservice/prep/engine.py | 5 +++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/iconservice/iiss/reward_calc/storage.py b/iconservice/iiss/reward_calc/storage.py index a52e88fff..a1e58cead 100644 --- a/iconservice/iiss/reward_calc/storage.py +++ b/iconservice/iiss/reward_calc/storage.py @@ -341,13 +341,11 @@ def get_total_elected_prep_delegated_snapshot(self) -> int: preps = data.prep_list break - if not preps: - raise InternalServiceErrorException(f"No PRepsData in iiss_data") - ret = 0 - for info in preps: - if info.address not in unreg_preps: - ret += info.value + if preps: + for info in preps: + if info.address not in unreg_preps: + ret += info.value Logger.info(tag=IISS_LOG_TAG, msg=f"get_total_elected_prep_delegated_snapshot load: {ret}") diff --git a/iconservice/prep/data/term.py b/iconservice/prep/data/term.py index 4201326df..8e7ffd71a 100644 --- a/iconservice/prep/data/term.py +++ b/iconservice/prep/data/term.py @@ -426,6 +426,9 @@ def from_list(cls, data: List, total_elected_prep_delegated_from_rc: int) -> 'Te total_elected_prep_delegated += delegated term._total_elected_prep_delegated = total_elected_prep_delegated + + if total_elected_prep_delegated_from_rc <= 0: + total_elected_prep_delegated_from_rc = total_elected_prep_delegated term._total_elected_prep_delegated_snapshot = total_elected_prep_delegated_from_rc term._generate_root_hash() diff --git a/iconservice/prep/engine.py b/iconservice/prep/engine.py index 5e565b0e0..fc7011a85 100644 --- a/iconservice/prep/engine.py +++ b/iconservice/prep/engine.py @@ -181,8 +181,7 @@ def rollback(self, context: 'IconScoreContext', _block_height: int, _block_hash: self.preps = self._load_preps(context) self.term = context.storage.prep.get_term(context) - - Logger.info(tag=ROLLBACK_LOG_TAG, msg="rollback() end") + Logger.info(tag=ROLLBACK_LOG_TAG, msg=f"rollback() end: {self.term}") def on_block_invoked( self, @@ -245,6 +244,8 @@ def _on_term_ended(self, context: 'IconScoreContext') -> Tuple[dict, 'Term']: main_preps_as_dict: dict = \ self._get_updated_main_preps(context, new_term, PRepResultState.NORMAL) + Logger.debug(tag=_TAG, msg=f"{new_term}") + return main_preps_as_dict, new_term @classmethod From 0ff42fea420e7977247a9d389090aa7a80a3749b Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Wed, 11 Dec 2019 21:17:56 +0900 Subject: [PATCH 22/36] Implement multi block rollback * Repeat to rollback up to a given block from the current block * Add a unittest for multi block rollback --- iconservice/database/wal.py | 3 + iconservice/icon_service_engine.py | 33 +++++----- iconservice/prep/data/term.py | 2 + iconservice/rollback/__init__.py | 24 +++++++ iconservice/rollback/rollback_manager.py | 27 +++----- .../iiss/decentralized/test_rollback.py | 66 +++++++++++++++++-- 6 files changed, 117 insertions(+), 38 deletions(-) diff --git a/iconservice/database/wal.py b/iconservice/database/wal.py index 2c97a240b..68087acc5 100644 --- a/iconservice/database/wal.py +++ b/iconservice/database/wal.py @@ -71,6 +71,9 @@ class WALState(Flag): # Send CALCULATE message to rc SEND_CALCULATE = auto() + # Means All flags are on + ALL = 0xFFFFFFFF + def tx_batch_value_to_bytes(tx_batch_value: 'TransactionBatchValue') -> Optional[bytes]: if not isinstance(tx_batch_value, TransactionBatchValue): diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 8f6a9a729..8fa2909c6 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -22,6 +22,7 @@ from iconservice.rollback.backup_cleaner import BackupCleaner from iconservice.rollback.backup_manager import BackupManager from iconservice.rollback.rollback_manager import RollbackManager +from iconservice.rollback import check_backup_exists from .base.address import Address, generate_score_address, generate_score_address_for_tbears from .base.address import ZERO_SCORE_ADDRESS, GOVERNANCE_SCORE_ADDRESS from .base.block import Block, EMPTY_BLOCK @@ -2003,24 +2004,24 @@ def remove_precommit_state(self, block_height: int, instant_block_hash: bytes) - Logger.warning(tag=self.TAG, msg="remove_precommit_state() end") def rollback(self, block_height: int, block_hash: bytes) -> dict: - """Rollback the current confirmed states to the old one indicated by block_height + """Rollback the current confirmed state to the old one indicated by block_height - :param block_height: - :param block_hash: + :param block_height: final block height after rollback + :param block_hash: final block hash after rollback :return: """ - Logger.warning(tag=ROLLBACK_LOG_TAG, - msg=f"rollback() start: height={block_height} hash={bytes_to_hex(block_hash)}") + Logger.info(tag=ROLLBACK_LOG_TAG, + msg=f"rollback() start: height={block_height} hash={bytes_to_hex(block_hash)}") last_block: 'Block' = self._get_last_block() Logger.info(tag=self.TAG, msg=f"last_block={last_block}") - # If rollback is impossible for the current status, + # If rollback is not possible for the current state, # self._is_rollback_needed() should raise an InternalServiceErrorException try: if self._is_rollback_needed(last_block, block_height, block_hash): context = self._context_factory.create(IconScoreContextType.DIRECT, block=last_block) - self._rollback(context, block_height, block_hash) + self._rollback(context, last_block, block_height, block_hash) except BaseException as e: Logger.error(tag=ROLLBACK_LOG_TAG, msg=str(e)) raise InternalServiceErrorException( @@ -2031,27 +2032,28 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: ConstantKeys.BLOCK_HASH: block_hash } - Logger.warning(tag=ROLLBACK_LOG_TAG, - msg=f"rollback() end: height={block_height}, hash={bytes_to_hex(block_hash)}") + Logger.info(tag=ROLLBACK_LOG_TAG, msg=f"rollback() end") return response - @classmethod - def _is_rollback_needed(cls, last_block: 'Block', block_height: int, block_hash: bytes) -> bool: + def _is_rollback_needed(self, last_block: 'Block', block_height: int, block_hash: bytes) -> bool: """Check if rollback is needed """ - if block_height == last_block.height - 1: - return True if block_height == last_block.height and block_hash == last_block.hash: + # No need to rollback return False + if check_backup_exists(self._backup_root_path, last_block.height, block_height): + # There are enough backup files to rollback + return True + raise InternalServiceErrorException( f"Failed to rollback: " f"height={block_height} " f"hash={bytes_to_hex(block_hash)} " f"last_block={last_block}") - def _rollback(self, context: 'IconScoreContext', block_height: int, block_hash: bytes): + def _rollback(self, context: 'IconScoreContext', last_block: 'Block', block_height: int, block_hash: bytes): # Close storage IconScoreContext.storage.rc.close() @@ -2060,7 +2062,8 @@ def _rollback(self, context: 'IconScoreContext', block_height: int, block_hash: # Rollback state_db and rc_data_db to those of a given block_height rollback_manager = RollbackManager(self._backup_root_path, self._rc_data_path) - rollback_manager.run(self._icx_context_db.key_value_db, block_height) + for height in range(last_block.height - 1, block_height - 1, -1): + rollback_manager.run(self._icx_context_db.key_value_db, height) # Clear all iconscores and reload builtin scores only builtin_score_owner: 'Address' = Address.from_string(self._conf[ConfigKey.BUILTIN_SCORE_OWNER]) diff --git a/iconservice/prep/data/term.py b/iconservice/prep/data/term.py index 8e7ffd71a..e5079547b 100644 --- a/iconservice/prep/data/term.py +++ b/iconservice/prep/data/term.py @@ -428,6 +428,8 @@ def from_list(cls, data: List, total_elected_prep_delegated_from_rc: int) -> 'Te term._total_elected_prep_delegated = total_elected_prep_delegated if total_elected_prep_delegated_from_rc <= 0: + # In the case of the first term (prevote -> decentralization), + # total_elected_prep_delegated_from_rc can be 0. total_elected_prep_delegated_from_rc = total_elected_prep_delegated term._total_elected_prep_delegated_snapshot = total_elected_prep_delegated_from_rc diff --git a/iconservice/rollback/__init__.py b/iconservice/rollback/__init__.py index aaa7b26b0..6dded5800 100644 --- a/iconservice/rollback/__init__.py +++ b/iconservice/rollback/__init__.py @@ -16,6 +16,8 @@ __all__ = "get_backup_filename" +import os + def get_backup_filename(block_height: int) -> str: """ @@ -24,3 +26,25 @@ def get_backup_filename(block_height: int) -> str: :return: """ return f"block-{block_height}.bak" + + +def check_backup_exists(backup_root_path: str, current_block_height: int, rollback_block_height: int) -> bool: + """Check if backup files for rollback exist + + :param backup_root_path: the directory where backup files are located + :param current_block_height: current state before rollback + :param rollback_block_height: final state after rollback + :return: True(exist) False(not exist) + """ + if current_block_height < 1 or \ + rollback_block_height < 0 or \ + rollback_block_height > current_block_height: + return False + + for block_height in range(current_block_height - 1, rollback_block_height - 1, -1): + filename = get_backup_filename(block_height) + path = os.path.join(backup_root_path, filename) + if not os.path.isfile(path): + return False + + return True diff --git a/iconservice/rollback/rollback_manager.py b/iconservice/rollback/rollback_manager.py index c9365a9f3..8ffdf69d5 100644 --- a/iconservice/rollback/rollback_manager.py +++ b/iconservice/rollback/rollback_manager.py @@ -19,13 +19,13 @@ from iconcommons.logger import Logger -from .backup_manager import WALBackupState, get_backup_filename +from iconservice.base.exception import DatabaseException from iconservice.database.db import KeyValueDatabase from iconservice.database.wal import WriteAheadLogReader, WALDBType -from iconservice.base.exception import DatabaseException from iconservice.icon_constant import ROLLBACK_LOG_TAG from iconservice.iiss.reward_calc import RewardCalcStorage from iconservice.iiss.reward_calc.msg_data import make_block_produce_info_key +from .backup_manager import WALBackupState, get_backup_filename if TYPE_CHECKING: from iconservice.database.db import KeyValueDatabase @@ -35,11 +35,15 @@ class RollbackManager(object): + """Rollback the current state to the one block previous one with a backup file + + """ + def __init__(self, backup_root_path: str, rc_data_path: str): self._backup_root_path = backup_root_path self._rc_data_path = rc_data_path - def run(self, icx_db: 'KeyValueDatabase', block_height: int = -1) -> Tuple[int, bool]: + def run(self, icx_db: 'KeyValueDatabase', block_height: int) -> Tuple[int, bool]: """Rollback to the previous block state Called on self.open() @@ -59,8 +63,6 @@ def run(self, icx_db: 'KeyValueDatabase', block_height: int = -1) -> Tuple[int, Logger.info(tag=TAG, msg=f"backup state file not found: {path}") return -1, False - block_height = -1 - is_calc_period_end_block = False reader = WriteAheadLogReader() try: @@ -68,6 +70,8 @@ def run(self, icx_db: 'KeyValueDatabase', block_height: int = -1) -> Tuple[int, is_calc_period_end_block = \ bool(WALBackupState(reader.state) & WALBackupState.CALC_PERIOD_END_BLOCK) + assert reader.block.height == block_height + if reader.log_count == 2: self._rollback_rc_db(reader, is_calc_period_end_block) self._rollback_state_db(reader, icx_db) @@ -75,6 +79,7 @@ def run(self, icx_db: 'KeyValueDatabase', block_height: int = -1) -> Tuple[int, except BaseException as e: Logger.debug(tag=TAG, msg=str(e)) + raise e finally: reader.close() @@ -158,18 +163,6 @@ def _rename_rc_db(cls, src_path: str, dst_path: str): def _rollback_state_db(cls, reader: 'WriteAheadLogReader', icx_db: 'KeyValueDatabase'): icx_db.write_batch(reader.get_iterator(WALDBType.STATE.value)) - def _clear_backup_files(self): - try: - with os.scandir(self._backup_root_path) as it: - for entry in it: - if entry.is_file() \ - and entry.name.startswith("block-") \ - and entry.name.endswith(".bak"): - path = os.path.join(self._backup_root_path, entry.name) - self._remove_backup_file(path) - except BaseException as e: - Logger.info(tag=TAG, msg=str(e)) - @classmethod def _remove_backup_file(cls, path: str): try: diff --git a/tests/integrate_test/iiss/decentralized/test_rollback.py b/tests/integrate_test/iiss/decentralized/test_rollback.py index 51ac688cb..72b6ecdbf 100644 --- a/tests/integrate_test/iiss/decentralized/test_rollback.py +++ b/tests/integrate_test/iiss/decentralized/test_rollback.py @@ -115,12 +115,6 @@ def setUp(self): self.assertEqual(expected_response, response) def test_rollback_icx_transfer(self): - """ - scenario 1 - when it starts new preps on new term, normal case, while 100 block. - expected : - all new preps have maintained until 100 block because it already passed GRACE_PERIOD - """ # Prevent icon_service_engine from sending RollbackRequest to rc IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() @@ -374,6 +368,66 @@ def test_rollback_set_delegation(self): self._check_if_last_block_is_reverted(prev_block) + def test_rollback_multi_blocks(self): + # Prevent icon_service_engine from sending RollbackRequest to rc + IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() + + # Inspect the current term + response = self.get_prep_term() + assert response["sequence"] == 2 + + prev_block: 'Block' = self.icon_service_engine._get_last_block() + + # Transfer 3000 icx to new 10 accounts + init_balance = icx_to_loop(3000) + accounts: List['EOAAccount'] = self.create_eoa_accounts(10) + self.distribute_icx(accounts=accounts, init_balance=init_balance) + + for account in accounts: + balance: int = self.get_balance(account.address) + assert balance == init_balance + + # accounts[0] transfers 10 ICX to receiver + sender: EOAAccount = accounts[0] + receiver: EOAAccount = self.create_eoa_account() + value = icx_to_loop(10) + tx = self.create_transfer_icx_tx(from_=sender.address, + to_=receiver.address, + value=value) + + # 2 == Base TX + ICX transfer TX + tx_results: List['TransactionResult'] = self.process_confirm_block(tx_list=[tx]) + assert len(tx_results) == 2 + + # ICX transfer TX success check + tx_result: 'TransactionResult' = tx_results[1] + assert tx_results[0].status == 1 + + # Sender balance check + sender_balance: int = self.get_balance(sender.address) + assert sender_balance == init_balance - value - tx_result.step_price * tx_result.step_used + + # Receiver balance check + receiver_balance: int = self.get_balance(receiver.address) + assert receiver_balance == value + + # Rollback the state to the previous block height + block: 'Block' = self.icon_service_engine._get_last_block() + assert prev_block.height == block.height - 2 + self._rollback(prev_block) + + # Check if the balances of accounts are reverted + for account in accounts: + balance: int = self.get_balance(account.address) + assert balance == 0 + + # Check if the balance of receiver is reverted to 0 + receiver_balance: int = self.get_balance(receiver.address) + assert receiver_balance == 0 + + # Check the last block + self._check_if_last_block_is_reverted(prev_block) + def _rollback(self, block: 'Block'): super().rollback(block.height, block.hash) self._check_if_rollback_reward_calculator_is_called(block) From b8a4675c2a79327a040d532b1db3489d0fe56331 Mon Sep 17 00:00:00 2001 From: boyeon555 Date: Mon, 16 Dec 2019 17:10:07 +0900 Subject: [PATCH 23/36] Add "getInactivePReps" on PREP_METHOD_TABLE --- iconservice/icon_constant.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iconservice/icon_constant.py b/iconservice/icon_constant.py index 38d75951a..997cc6bb8 100644 --- a/iconservice/icon_constant.py +++ b/iconservice/icon_constant.py @@ -121,7 +121,6 @@ class Revision(Enum): RC_DB_VERSION_0 = 0 RC_DB_VERSION_2 = 2 - # The case that version is updated but not revision, set the version to the current revision # The case that both version and revision is updated, add revision field to the version table # The case that only revision is changed, do not update this table @@ -252,7 +251,8 @@ class DeployState(IntEnum): "getMainPReps", "getSubPReps", "getPReps", - "getPRepTerm" + "getPRepTerm", + "getInactivePReps" ] DEBUG_METHOD_TABLE = [ @@ -292,8 +292,8 @@ class DeployState(IntEnum): PENALTY_GRACE_PERIOD = IISS_DAY_BLOCK * 2 -LOW_PRODUCTIVITY_PENALTY_THRESHOLD = 85 # Unit: Percent -BLOCK_VALIDATION_PENALTY_THRESHOLD = 660 # Unit: Blocks +LOW_PRODUCTIVITY_PENALTY_THRESHOLD = 85 # Unit: Percent +BLOCK_VALIDATION_PENALTY_THRESHOLD = 660 # Unit: Blocks BASE_TRANSACTION_VERSION = 3 From 0da1a31a73aae0177f5cc96b0b933bc1659cce5f Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Mon, 16 Dec 2019 19:39:58 +0900 Subject: [PATCH 24/36] Apply write_batch to RollbackManager * Consider the case that there is a term change during rollback * Fix a minor bug in rollback/test_backup_manager.py --- iconservice/icon_service_engine.py | 27 +++- iconservice/rollback/rollback_manager.py | 191 ++++++++++------------- tests/rollback/test_backup_manager.py | 17 +- 3 files changed, 110 insertions(+), 125 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 8fa2909c6..9d30b6149 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -19,10 +19,10 @@ from iconcommons.logger import Logger +from iconservice.rollback import check_backup_exists from iconservice.rollback.backup_cleaner import BackupCleaner from iconservice.rollback.backup_manager import BackupManager from iconservice.rollback.rollback_manager import RollbackManager -from iconservice.rollback import check_backup_exists from .base.address import Address, generate_score_address, generate_score_address_for_tbears from .base.address import ZERO_SCORE_ADDRESS, GOVERNANCE_SCORE_ADDRESS from .base.block import Block, EMPTY_BLOCK @@ -2053,17 +2053,28 @@ def _is_rollback_needed(self, last_block: 'Block', block_height: int, block_hash f"hash={bytes_to_hex(block_hash)} " f"last_block={last_block}") - def _rollback(self, context: 'IconScoreContext', last_block: 'Block', block_height: int, block_hash: bytes): + def _rollback(self, context: 'IconScoreContext', + last_block: 'Block', rollback_block_height: int, rollback_block_hash: bytes): + """ + + :param context: + :param last_block: + :param rollback_block_height: final block_height after rollback + :param rollback_block_hash: final block hash after rollback + """ # Close storage IconScoreContext.storage.rc.close() # Rollback the state of reward_calculator prior to iconservice - IconScoreContext.engine.iiss.rollback_reward_calculator(block_height, block_hash) + IconScoreContext.engine.iiss.rollback_reward_calculator(rollback_block_height, rollback_block_hash) # Rollback state_db and rc_data_db to those of a given block_height - rollback_manager = RollbackManager(self._backup_root_path, self._rc_data_path) - for height in range(last_block.height - 1, block_height - 1, -1): - rollback_manager.run(self._icx_context_db.key_value_db, height) + rollback_manager = RollbackManager( + self._backup_root_path, self._rc_data_path, self._icx_context_db.key_value_db) + rollback_manager.run( + current_block_height=last_block.height, + rollback_block_height=rollback_block_height, + start_block_height_in_term=IconScoreContext.engine.prep.term.start_block_height) # Clear all iconscores and reload builtin scores only builtin_score_owner: 'Address' = Address.from_string(self._conf[ConfigKey.BUILTIN_SCORE_OWNER]) @@ -2082,7 +2093,7 @@ def _rollback(self, context: 'IconScoreContext', last_block: 'Block', block_heig IconScoreContext.storage.rc, ] for storage in storages: - storage.rollback(context, block_height, block_hash) + storage.rollback(context, rollback_block_height, rollback_block_hash) # Rollback engines to block_height engines = [ @@ -2094,7 +2105,7 @@ def _rollback(self, context: 'IconScoreContext', last_block: 'Block', block_heig IconScoreContext.engine.issue, ] for engine in engines: - engine.rollback(context, block_height, block_hash) + engine.rollback(context, rollback_block_height, rollback_block_hash) self._init_last_block_info(context) diff --git a/iconservice/rollback/rollback_manager.py b/iconservice/rollback/rollback_manager.py index 8ffdf69d5..a8014ee8b 100644 --- a/iconservice/rollback/rollback_manager.py +++ b/iconservice/rollback/rollback_manager.py @@ -14,18 +14,17 @@ # limitations under the License. import os -import shutil -from typing import TYPE_CHECKING, Tuple +from typing import TYPE_CHECKING, Iterable, Optional, Tuple from iconcommons.logger import Logger -from iconservice.base.exception import DatabaseException -from iconservice.database.db import KeyValueDatabase -from iconservice.database.wal import WriteAheadLogReader, WALDBType -from iconservice.icon_constant import ROLLBACK_LOG_TAG -from iconservice.iiss.reward_calc import RewardCalcStorage -from iconservice.iiss.reward_calc.msg_data import make_block_produce_info_key -from .backup_manager import WALBackupState, get_backup_filename +from .backup_manager import get_backup_filename +from ..base.exception import InvalidParamsException, InternalServiceErrorException +from ..database.db import KeyValueDatabase +from ..database.wal import WriteAheadLogReader, WALDBType +from ..icon_constant import ROLLBACK_LOG_TAG +from ..iiss.reward_calc import RewardCalcStorage +from ..iiss.reward_calc.msg_data import make_block_produce_info_key if TYPE_CHECKING: from iconservice.database.db import KeyValueDatabase @@ -37,58 +36,85 @@ class RollbackManager(object): """Rollback the current state to the one block previous one with a backup file + Assume that the rollback of Reward Calculator has been already done + Related databases: state_db, iiss_db """ - def __init__(self, backup_root_path: str, rc_data_path: str): + def __init__(self, backup_root_path: str, rc_data_path: str, state_db: 'KeyValueDatabase'): self._backup_root_path = backup_root_path self._rc_data_path = rc_data_path + self._state_db = state_db - def run(self, icx_db: 'KeyValueDatabase', block_height: int) -> Tuple[int, bool]: + def run(self, current_block_height: int, rollback_block_height: int, start_block_height_in_term: int): """Rollback to the previous block state - Called on self.open() - - :param icx_db: state db - :param block_height: the height of block to rollback to - :return: the height of block after rollback, is_calc_period_end_block + :param current_block_height: + :param rollback_block_height: the height of block to rollback to + :param start_block_height_in_term: """ - Logger.info(tag=TAG, msg=f"rollback() start: BH={block_height}") - - if block_height < 0: - Logger.debug(tag=TAG, msg="rollback() end") - return -1, False + Logger.info(tag=TAG, msg=f"run() start: " + f"current_block_height={current_block_height} " + f"rollback_block_height={rollback_block_height}") - path: str = self._get_backup_file_path(block_height) - if not os.path.isfile(path): - Logger.info(tag=TAG, msg=f"backup state file not found: {path}") - return -1, False + self._validate_block_heights(current_block_height, rollback_block_height) + # Check whether a term change exists between current_block_height -1 and rollback_block_height, inclusive + term_change_exists = rollback_block_height < start_block_height_in_term + end_calc_block_height = start_block_height_in_term - 1 reader = WriteAheadLogReader() + state_db_batch = {} + iiss_db_batch = {} + + for block_height in range(current_block_height - 1, rollback_block_height - 1, -1): + # Make backup file with a given block_height + path: str = self._get_backup_file_path(block_height) + if not os.path.isfile(path): + raise InternalServiceErrorException(f"Backup file not found: {path}") - try: reader.open(path) - is_calc_period_end_block = \ - bool(WALBackupState(reader.state) & WALBackupState.CALC_PERIOD_END_BLOCK) - assert reader.block.height == block_height + # Merge backup data into state_db_batch + self._write_batch(reader.get_iterator(WALDBType.STATE.value), state_db_batch) - if reader.log_count == 2: - self._rollback_rc_db(reader, is_calc_period_end_block) - self._rollback_state_db(reader, icx_db) - block_height = reader.block.height + # Merge backup data into iiss_db_batch + if not (term_change_exists and block_height > end_calc_block_height): + self._write_batch(reader.get_iterator(WALDBType.RC.value), iiss_db_batch) - except BaseException as e: - Logger.debug(tag=TAG, msg=str(e)) - raise e - finally: reader.close() - # Remove the backup file after rollback is done - self._remove_backup_file(path) + # If a term change is detected during rollback, handle the exceptions below + if term_change_exists: + self._remove_block_produce_info(iiss_db_batch, end_calc_block_height) + self._rename_iiss_db(end_calc_block_height) + + # Commit write_batch to db + self._commit_batch(state_db_batch, self._state_db) + iiss_db = RewardCalcStorage.create_current_db(self._rc_data_path) + self._commit_batch(iiss_db_batch, iiss_db) + iiss_db.close() + + Logger.info(tag=TAG, msg="run() end") + + @staticmethod + def _validate_block_heights(current_block_height: int, rollback_block_height: int): + if current_block_height < 0: + raise InvalidParamsException(f"Invalid currentBlockHeight: {current_block_height}") - Logger.info(tag=TAG, msg=f"rollback() end: return={block_height}, {is_calc_period_end_block}") + if rollback_block_height < 0: + raise InvalidParamsException(f"Invalid rollbackBlockHeight: {rollback_block_height}") - return block_height, is_calc_period_end_block + if current_block_height <= rollback_block_height: + raise InvalidParamsException( + f"currentBlockHeight({current_block_height}) <= rollbackBlockHeight({rollback_block_height}") + + @staticmethod + def _write_batch(it: Iterable[Tuple[bytes, Optional[bytes]]], batch: dict): + for key, value in it: + batch[key] = value + + @staticmethod + def _commit_batch(batch: dict, db: 'KeyValueDatabase'): + db.write_batch(batch.items()) def _get_backup_file_path(self, block_height: int) -> str: """ @@ -101,68 +127,6 @@ def _get_backup_file_path(self, block_height: int) -> str: filename = get_backup_filename(block_height) return os.path.join(self._backup_root_path, filename) - def _rollback_rc_db(self, reader: 'WriteAheadLogReader', is_calc_period_end_block: bool): - """Rollback the state of rc_db to the previous one - - :param reader: - :param is_calc_period_end_block: - :return: - """ - Logger.debug(tag=TAG, msg=f"_rollback_rc_db() start: is_end_block={is_calc_period_end_block}") - - if is_calc_period_end_block: - Logger.info(tag=TAG, msg=f"BH-{reader.block.height} is a calc period end block") - self._rollback_rc_db_on_end_block(reader) - else: - db: 'KeyValueDatabase' = RewardCalcStorage.create_current_db(self._rc_data_path) - db.write_batch(reader.get_iterator(WALDBType.RC.value)) - db.close() - - Logger.debug(tag=TAG, msg=f"_rollback_rc_db() end") - - def _rollback_rc_db_on_end_block(self, reader: 'WriteAheadLogReader'): - """ - - :param reader: - :return: - """ - Logger.debug(tag=TAG, msg=f"_rollback_rc_db_on_end_block() start") - - current_rc_db_path, standby_rc_db_path, iiss_rc_db_path = \ - RewardCalcStorage.scan_rc_db(self._rc_data_path) - # Assume that standby_rc_db does not exist - assert standby_rc_db_path == "" - - current_rc_db_exists = len(current_rc_db_path) > 0 - iiss_rc_db_exists = len(iiss_rc_db_path) > 0 - - if current_rc_db_exists: - if iiss_rc_db_exists: - # Remove the next calc_period current_rc_db and rename iiss_rc_db to current_rc_db - shutil.rmtree(current_rc_db_path) - self._rename_rc_db(iiss_rc_db_path, current_rc_db_path) - else: - if iiss_rc_db_exists: - # iiss_rc_db -> current_rc_db - self._rename_rc_db(iiss_rc_db_path, current_rc_db_path) - else: - # If both current_rc_db and iiss_rc_db do not exist, raise error - raise DatabaseException(f"RC DB not found") - - self._remove_block_produce_info(current_rc_db_path, reader.block.height) - - Logger.debug(tag=TAG, msg=f"_rollback_rc_db_on_end_block() end") - - @classmethod - def _rename_rc_db(cls, src_path: str, dst_path: str): - Logger.info(tag=TAG, msg=f"_rename_rc_db() start: src={src_path} dst={dst_path}") - os.rename(src_path, dst_path) - Logger.info(tag=TAG, msg=f"_rename_rc_db() end") - - @classmethod - def _rollback_state_db(cls, reader: 'WriteAheadLogReader', icx_db: 'KeyValueDatabase'): - icx_db.write_batch(reader.get_iterator(WALDBType.STATE.value)) - @classmethod def _remove_backup_file(cls, path: str): try: @@ -172,21 +136,26 @@ def _remove_backup_file(cls, path: str): except BaseException as e: Logger.debug(tag=TAG, msg=str(e)) + def _rename_iiss_db(self, end_calc_block_height: int): + src_path = os.path.join( + self._rc_data_path, f"{RewardCalcStorage.IISS_RC_DB_NAME_PREFIX}{end_calc_block_height}") + dst_path = os.path.join(self._rc_data_path, RewardCalcStorage.CURRENT_IISS_DB_NAME) + + os.rename(src_path, dst_path) + @classmethod - def _remove_block_produce_info(cls, db_path: str, block_height: int): + def _remove_block_produce_info(cls, iiss_db_batch: dict, block_height: int): """Remove block_produce_info of calc_period_end_block from current_db - :param db_path: - :param block_height: + :param iiss_db_batch: + :param block_height: the end block of the previous term :return: """ Logger.debug(tag=TAG, - msg=f"_remove_block_produce_info() start: db_path={db_path} block_height={block_height}") + msg=f"_remove_block_produce_info() start: block_height={block_height}") + # Remove the end calc block from iiss_db key: bytes = make_block_produce_info_key(block_height) - - db = KeyValueDatabase.from_path(db_path, create_if_missing=False) - db.delete(key) - db.close() + iiss_db_batch[key] = None Logger.debug(tag=TAG, msg="_remove_block_produce_info() end") diff --git a/tests/rollback/test_backup_manager.py b/tests/rollback/test_backup_manager.py index 1ee7e202a..d02b61d5b 100644 --- a/tests/rollback/test_backup_manager.py +++ b/tests/rollback/test_backup_manager.py @@ -71,7 +71,8 @@ def setUp(self) -> None: self.rollback_manager = RollbackManager( backup_root_path=backup_root_path, - rc_data_path=rc_data_path + rc_data_path=rc_data_path, + state_db=icx_db ) self.state_db_root_path = state_db_root_path @@ -185,8 +186,12 @@ def _check_if_rollback_is_done(db: 'KeyValueDatabase', prev_state: OrderedDict): def _rollback_with_rollback_manager(self, last_block: 'Block'): rollback_manager = self.rollback_manager - - block_height, is_calc_end_block_height = \ - rollback_manager.run(self.state_db, last_block.height) - assert block_height == last_block.height - assert not is_calc_end_block_height + current_block_height = last_block.height + 1 + rollback_block_height = last_block.height + + # One block rollback + ret = rollback_manager.run( + current_block_height, + rollback_block_height, + start_block_height_in_term=rollback_block_height - 1) + assert ret is None From c96946b8b6396ffc2ec29820db4d46f392e1a51c Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Wed, 18 Dec 2019 14:26:27 +0900 Subject: [PATCH 25/36] Remove rc_version from iiss_rc_db name * Fix minor unittest errors caused by iiss_rc_db name change --- iconservice/icon_service_engine.py | 13 ++----- iconservice/iiss/reward_calc/storage.py | 14 ++++---- iconservice/rollback/rollback_manager.py | 9 +++++ .../iiss/decentralized/test_rc_db_data.py | 11 +++--- .../iiss/test_recover_using_wal.py | 36 ++++++++++++------- 5 files changed, 46 insertions(+), 37 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 9d30b6149..28a35559b 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -62,7 +62,7 @@ from .icx.issue.base_transaction_creator import BaseTransactionCreator from .iiss import IISSEngine, IISSStorage, check_decentralization_condition from .iiss.reward_calc import RewardCalcStorage, RewardCalcDataCreator -from .iiss.reward_calc.storage import RewardCalcDBInfo, get_version_and_revision +from .iiss.reward_calc.storage import RewardCalcDBInfo from .inner_call import inner_call from .meta import MetaDBStorage from .precommit_data_manager import PrecommitData, PrecommitDataManager, PrecommitFlag @@ -2209,20 +2209,13 @@ def _recover_rc_db(cls, reader: 'WriteAheadLogReader', rc_data_path: str): # If only current_db exists, replace current_db to standby_rc_db if is_current_exists and not is_standby_exists and not is_iiss_exists: - # Get revision from the RC DB - prev_calc_db: 'KeyValueDatabase' = RewardCalcStorage.create_current_db(rc_data_path) - rc_version, revision = get_version_and_revision(prev_calc_db) - rc_version: int = max(rc_version, 0) - prev_calc_db.close() - # Process compaction before send the RC DB to reward calculator prev_calc_db_path: str = os.path.join(rc_data_path, RewardCalcStorage.CURRENT_IISS_DB_NAME) RewardCalcStorage.process_db_compaction(prev_calc_db_path) calculate_block_height: int = reader.block.height - 1 - standby_rc_db_path: str = RewardCalcStorage.rename_current_db_to_standby_db(rc_data_path, - calculate_block_height, - rc_version) + standby_rc_db_path: str = \ + RewardCalcStorage.rename_current_db_to_standby_db(rc_data_path, calculate_block_height) is_standby_exists: bool = True elif not is_current_exists and not is_standby_exists: # No matter iiss_db exists or not, If both current_db and standby_db do not exist, raise error diff --git a/iconservice/iiss/reward_calc/storage.py b/iconservice/iiss/reward_calc/storage.py index a1e58cead..ec6a8a58b 100644 --- a/iconservice/iiss/reward_calc/storage.py +++ b/iconservice/iiss/reward_calc/storage.py @@ -143,8 +143,8 @@ def _supplement_db(self, context: 'IconScoreContext', revision: int): Logger.debug(tag=IISS_LOG_TAG, msg=f"No header data. Put Header to db on open: {str(header)}") @classmethod - def get_standby_rc_db_name(cls, block_height: int, rc_version: int) -> str: - return f"{cls.STANDBY_IISS_DB_NAME_PREFIX}{block_height}_{rc_version}" + def get_standby_rc_db_name(cls, block_height: int) -> str: + return f"{cls.STANDBY_IISS_DB_NAME_PREFIX}{block_height}" def put_data_directly(self, iiss_data: 'Data', tx_index: Optional[int] = None): if isinstance(iiss_data, TxData): @@ -230,7 +230,7 @@ def _rename_db(old_db_path: str, new_db_path: str): def replace_db(self, block_height: int) -> 'RewardCalcDBInfo': """ - 1. Rename current_db to standby_db_{block_height}_{rc_version} + 1. Rename current_db to standby_db_{block_height} 2. Create a new current_db for the next calculation period :param block_height: End block height of the current calc period @@ -240,13 +240,11 @@ def replace_db(self, block_height: int) -> 'RewardCalcDBInfo': # rename current db -> standby db assert block_height > 0 - rc_version, _ = self.get_version_and_revision() - rc_version: int = max(rc_version, 0) self._db.close() # Process compaction before send the RC DB to reward calculator self.process_db_compaction(os.path.join(self._path, self.CURRENT_IISS_DB_NAME)) - standby_db_path: str = self.rename_current_db_to_standby_db(self._path, block_height, rc_version) + standby_db_path: str = self.rename_current_db_to_standby_db(self._path, block_height) self._db = self.create_current_db(self._path) return RewardCalcDBInfo(standby_db_path, block_height) @@ -270,9 +268,9 @@ def create_current_db(cls, rc_data_path: str) -> 'KeyValueDatabase': return KeyValueDatabase.from_path(current_db_path, create_if_missing=True) @classmethod - def rename_current_db_to_standby_db(cls, rc_data_path: str, block_height: int, rc_version: int) -> str: + def rename_current_db_to_standby_db(cls, rc_data_path: str, block_height: int) -> str: current_db_path: str = os.path.join(rc_data_path, cls.CURRENT_IISS_DB_NAME) - standby_db_name: str = cls.get_standby_rc_db_name(block_height, rc_version) + standby_db_name: str = cls.get_standby_rc_db_name(block_height) standby_db_path: str = os.path.join(rc_data_path, standby_db_name) cls._rename_db(current_db_path, standby_db_path) diff --git a/iconservice/rollback/rollback_manager.py b/iconservice/rollback/rollback_manager.py index a8014ee8b..53a24fe9f 100644 --- a/iconservice/rollback/rollback_manager.py +++ b/iconservice/rollback/rollback_manager.py @@ -14,6 +14,7 @@ # limitations under the License. import os +import shutil from typing import TYPE_CHECKING, Iterable, Optional, Tuple from iconcommons.logger import Logger @@ -137,12 +138,20 @@ def _remove_backup_file(cls, path: str): Logger.debug(tag=TAG, msg=str(e)) def _rename_iiss_db(self, end_calc_block_height: int): + Logger.debug(tag=TAG, msg=f"_rename_iiss_db() start: end_calc_block_height={end_calc_block_height}") + src_path = os.path.join( self._rc_data_path, f"{RewardCalcStorage.IISS_RC_DB_NAME_PREFIX}{end_calc_block_height}") dst_path = os.path.join(self._rc_data_path, RewardCalcStorage.CURRENT_IISS_DB_NAME) + Logger.info(tag=TAG, msg=f"rename_iiss_db: src_path={src_path} dst_path={dst_path}") + # Remove a new current_db + shutil.rmtree(dst_path, ignore_errors=True) + # Rename iiss_rc_db_{BH} to current_db os.rename(src_path, dst_path) + Logger.debug(tag=TAG, msg="_rename_iiss_db() end") + @classmethod def _remove_block_produce_info(cls, iiss_db_batch: dict, block_height: int): """Remove block_produce_info of calc_period_end_block from current_db diff --git a/tests/integrate_test/iiss/decentralized/test_rc_db_data.py b/tests/integrate_test/iiss/decentralized/test_rc_db_data.py index 383b983b3..370852857 100644 --- a/tests/integrate_test/iiss/decentralized/test_rc_db_data.py +++ b/tests/integrate_test/iiss/decentralized/test_rc_db_data.py @@ -43,8 +43,8 @@ def get_last_rc_db_data(rc_data_path): key=lambda rc_dir: int(rc_dir[len(RewardCalcStorage.IISS_RC_DB_NAME_PREFIX):]), reverse=True)[0] - def _check_the_name_of_rc_db(self, actual_rc_db_name: str, version: int = 0): - expected_last_rc_db_name: str = Storage.IISS_RC_DB_NAME_PREFIX + str(self._block_height - 1) + '_' + str(version) + def _check_the_name_of_rc_db(self, actual_rc_db_name: str): + expected_last_rc_db_name: str = Storage.IISS_RC_DB_NAME_PREFIX + str(self._block_height - 1) self.assertEqual(expected_last_rc_db_name, actual_rc_db_name) def test_all_rc_db_data_block_height(self): @@ -59,7 +59,7 @@ def test_all_rc_db_data_block_height(self): self.make_blocks(self._block_height + 1) get_last_rc_db: str = self.get_last_rc_db_data(rc_data_path) expected_version: int = 0 - self._check_the_name_of_rc_db(get_last_rc_db, expected_version) + self._check_the_name_of_rc_db(get_last_rc_db) rc_db = KeyValueDatabase.from_path(os.path.join(rc_data_path, get_last_rc_db)) expected_rc_db_data_count: int = 1 @@ -164,7 +164,7 @@ def test_all_rc_db_data_block_height(self): prev_block_validators=main_preps_address[1:]) get_last_rc_db: str = self.get_last_rc_db_data(rc_data_path) expected_version: int = 0 - self._check_the_name_of_rc_db(get_last_rc_db, expected_version) + self._check_the_name_of_rc_db(get_last_rc_db) rc_db = KeyValueDatabase.from_path(os.path.join(rc_data_path, get_last_rc_db)) self.assertIsNotNone(rc_db.get(Header.PREFIX)) for rc_data in rc_db.iterator(): @@ -207,8 +207,7 @@ def test_all_rc_db_data_block_height(self): prev_block_validators=main_preps_address[1:]) expected_pr_block_height: int = expected_gv_block_height get_last_rc_db: str = self.get_last_rc_db_data(rc_data_path) - expected_version: int = 2 - self._check_the_name_of_rc_db(get_last_rc_db, expected_version) + self._check_the_name_of_rc_db(get_last_rc_db) rc_db = KeyValueDatabase.from_path(os.path.join(rc_data_path, get_last_rc_db)) expected_bp_block_height: int = expected_gv_block_height + 1 diff --git a/tests/integrate_test/iiss/test_recover_using_wal.py b/tests/integrate_test/iiss/test_recover_using_wal.py index 0681f57e6..73d7e248e 100644 --- a/tests/integrate_test/iiss/test_recover_using_wal.py +++ b/tests/integrate_test/iiss/test_recover_using_wal.py @@ -17,6 +17,7 @@ import os import shutil from enum import IntFlag, auto +from typing import TYPE_CHECKING from iconservice.database.db import KeyValueDatabase from iconservice.database.wal import WriteAheadLogWriter, StateWAL, IissWAL, WALState @@ -30,6 +31,9 @@ from tests.integrate_test.iiss.test_iiss_base import TestIISSBase from tests.integrate_test.test_integrate_base import EOAAccount +if TYPE_CHECKING: + from iconservice.base.block import Block + class RCDataCheckFlag(IntFlag): NONE = 0 @@ -97,8 +101,8 @@ def _get_wal_writer(self, precommit_data: 'PrecommitData', is_calc_period_start_ iiss_wal: 'IissWAL' = IissWAL(precommit_data.rc_block_batch, tx_index, revision) return wal_writer, state_wal, iiss_wal - def _write_batch_to_wal(self, - wal_writer: 'WriteAheadLogWriter', + @staticmethod + def _write_batch_to_wal(wal_writer: 'WriteAheadLogWriter', state_wal: 'StateWAL', iiss_wal: 'IissWAL', is_calc_period_start_block: bool): @@ -109,10 +113,10 @@ def _write_batch_to_wal(self, wal_writer.write_walogable(state_wal) wal_writer.flush() - def _get_last_block_from_icon_service(self) -> 'Block': + def _get_last_block_from_icon_service(self) -> int: return self.icon_service_engine._get_last_block().height - def _get_commit_context(self, block: 'block'): + def _get_commit_context(self, block: 'Block'): return self.icon_service_engine._context_factory.create(IconScoreContextType.DIRECT, block) def _close_and_reopen_iconservice(self): @@ -132,9 +136,9 @@ def _check_the_state_and_rc_db_after_recover(self, block_height: int, is_calc_pe # Check if rc db is updated rc_data_path: str = os.path.join(self._state_db_root_path, IISS_DB) # Get_last_rc_db: str = TestRCDatabase.get_last_rc_db_data(rc_data_path) - cuerent_rc_db = KeyValueDatabase.from_path(os.path.join(rc_data_path, "current_db")) + current_rc_db = KeyValueDatabase.from_path(os.path.join(rc_data_path, "current_db")) rc_data_flag = RCDataCheckFlag(0) - for rc_data in cuerent_rc_db.iterator(): + for rc_data in current_rc_db.iterator(): if rc_data[0][:2] == PRepsData.PREFIX: pr: 'PRepsData' = PRepsData.from_bytes(rc_data[0], rc_data[1]) if pr.block_height == block_height: @@ -164,8 +168,11 @@ def _check_the_state_and_rc_db_after_recover(self, block_height: int, is_calc_pe if is_calc_period_start_block: self.assertEqual(RCDataCheckFlag.ALL_ON_START, rc_data_flag) - self.assertTrue(os.path.exists(os.path.join(rc_data_path, - f"{RewardCalcStorage.IISS_RC_DB_NAME_PREFIX}{block_height - 1}_2"))) + self.assertTrue( + os.path.isdir( + os.path.join(rc_data_path, f"{RewardCalcStorage.IISS_RC_DB_NAME_PREFIX}{block_height - 1}") + ) + ) else: self.assertEqual(RCDataCheckFlag.ALL_ON_CALC, rc_data_flag) @@ -208,6 +215,7 @@ def test_close_during_writing_rc_db_on_calc_period(self): context: 'IconScoreContext' = self._get_commit_context(precommit_data.block) wal_writer, state_wal, iiss_wal = self._get_wal_writer(precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) + wal_writer.close() # write rc data to rc db # do not write state of wal (which means overwriting the rc data to db) @@ -231,6 +239,7 @@ def test_close_after_writing_rc_db_on_calc_period(self): # write rc data to rc db self.icon_service_engine._process_iiss_commit(context, precommit_data, iiss_wal, is_start_block) wal_writer.write_state(WALState.WRITE_RC_DB.value, add=True) + wal_writer.close() self._close_and_reopen_iconservice() @@ -246,6 +255,7 @@ def test_close_before_change_current_to_standby_on_the_start(self): precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke() wal_writer, state_wal, iiss_wal = self._get_wal_writer(precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) + wal_writer.close() # remove all iiss_db self._remove_all_iiss_db_before_reopen() @@ -259,13 +269,13 @@ def test_close_only_standby_exists_on_the_start(self): self.make_blocks_to_end_calculation() is_start_block: bool = True last_block_before_close: int = self._get_last_block_from_icon_service() - rc_version: int = 2 precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke() wal_writer, state_wal, iiss_wal = self._get_wal_writer(precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) + wal_writer.close() # Change the current_db to standby_db - RewardCalcStorage.rename_current_db_to_standby_db(self.rc_data_path, last_block_before_close, rc_version) + RewardCalcStorage.rename_current_db_to_standby_db(self.rc_data_path, last_block_before_close) # Remove all iiss_db self._remove_all_iiss_db_before_reopen() @@ -279,13 +289,13 @@ def test_close_standby_and_current_exists_on_the_start(self): self.make_blocks_to_end_calculation() is_start_block: bool = True last_block_before_close: int = self._get_last_block_from_icon_service() - rc_version: int = 2 precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke() wal_writer, state_wal, iiss_wal = self._get_wal_writer(precommit_data, is_start_block) self._write_batch_to_wal(wal_writer, state_wal, iiss_wal, is_start_block) + wal_writer.close() # Change the current_db to standby_db - RewardCalcStorage.rename_current_db_to_standby_db(self.rc_data_path, last_block_before_close, rc_version) + RewardCalcStorage.rename_current_db_to_standby_db(self.rc_data_path, last_block_before_close) RewardCalcStorage.create_current_db(self.rc_data_path) # Remove all iiss_db @@ -300,7 +310,6 @@ def test_close_before_sending_calculate_on_the_start(self): self.make_blocks_to_end_calculation() is_start_block: bool = True last_block_before_close: int = self._get_last_block_from_icon_service() - rc_version: int = 2 precommit_data: 'PrecommitData' = self._get_precommit_data_after_invoke() context: 'IconScoreContext' = self._get_commit_context(precommit_data.block) @@ -319,6 +328,7 @@ def test_close_before_sending_calculate_on_the_start(self): self.icon_service_engine._process_state_commit(context, precommit_data, state_wal) wal_writer.write_state(WALState.WRITE_STATE_DB.value, add=True) wal_writer.flush() + wal_writer.close() # Remove all iiss_db self._remove_all_iiss_db_before_reopen() From 683489d7c0965491bb933e434f66a3cde4291182 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Thu, 19 Dec 2019 11:05:05 +0900 Subject: [PATCH 26/36] Implement rollback recovery * Record Rollback metadata before rollback for rollback recovery * Add Metadata class * Add unittests --- iconservice/icon_service_engine.py | 75 +++++++++++++-- iconservice/rollback/metadata.py | 91 +++++++++++++++++++ iconservice/rollback/rollback_manager.py | 44 +++++---- .../iiss/decentralized/test_rollback.py | 56 +++++++++++- tests/integrate_test/iiss/test_iiss_base.py | 11 ++- tests/rollback/test_backup_manager.py | 6 +- tests/rollback/test_metadata.py | 69 ++++++++++++++ 7 files changed, 319 insertions(+), 33 deletions(-) create mode 100644 iconservice/rollback/metadata.py create mode 100644 tests/rollback/test_metadata.py diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 28a35559b..23f692fd5 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -72,6 +72,7 @@ from .utils import sha3_256, int_to_bytes, ContextEngine, ContextStorage from .utils import to_camel_case, bytes_to_hex from .utils.bloom import BloomFilter +from .rollback.metadata import Metadata as RollbackMetadata if TYPE_CHECKING: from .iconscore.icon_score_event_log import EventLog @@ -91,6 +92,7 @@ class IconServiceEngine(ContextContainer): """ TAG = "ISE" WAL_FILE = "block.wal" + ROLLBACK_METADATA_FILE = "ROLLBACK_METADATA" def __init__(self): """Constructor @@ -173,6 +175,7 @@ def open(self, conf: 'IconConfig'): IconScoreContext.precommitdata_log_flag = conf[ConfigKey.PRECOMMIT_DATA_LOG_FLAG] self._init_component_context() + # Recover incomplete state on wal and rollback process self._recover_dbs(rc_data_path) # load last_block_info @@ -2020,8 +2023,20 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: # self._is_rollback_needed() should raise an InternalServiceErrorException try: if self._is_rollback_needed(last_block, block_height, block_hash): + # Get the start block height of this term + term_start_block_height: int = IconScoreContext.engine.prep.term.start_block_height + + # Record rollback metadata + path = self._get_rollback_metadata_path() + with open(path, "wb") as f: + metadata = RollbackMetadata( + block_height, block_hash, term_start_block_height, last_block) + f.write(metadata.to_bytes()) + + # Do rollback context = self._context_factory.create(IconScoreContextType.DIRECT, block=last_block) - self._rollback(context, last_block, block_height, block_hash) + self._rollback(context, block_height, block_hash, term_start_block_height) + except BaseException as e: Logger.error(tag=ROLLBACK_LOG_TAG, msg=str(e)) raise InternalServiceErrorException( @@ -2054,11 +2069,12 @@ def _is_rollback_needed(self, last_block: 'Block', block_height: int, block_hash f"last_block={last_block}") def _rollback(self, context: 'IconScoreContext', - last_block: 'Block', rollback_block_height: int, rollback_block_hash: bytes): + rollback_block_height: int, + rollback_block_hash: bytes, + term_start_block_height: int): """ :param context: - :param last_block: :param rollback_block_height: final block_height after rollback :param rollback_block_hash: final block hash after rollback """ @@ -2072,9 +2088,9 @@ def _rollback(self, context: 'IconScoreContext', rollback_manager = RollbackManager( self._backup_root_path, self._rc_data_path, self._icx_context_db.key_value_db) rollback_manager.run( - current_block_height=last_block.height, + last_block_height=context.block.height, rollback_block_height=rollback_block_height, - start_block_height_in_term=IconScoreContext.engine.prep.term.start_block_height) + term_start_block_height=term_start_block_height) # Clear all iconscores and reload builtin scores only builtin_score_owner: 'Address' = Address.from_string(self._conf[ConfigKey.BUILTIN_SCORE_OWNER]) @@ -2137,16 +2153,56 @@ def inner_call(self, request: dict): return inner_call(context, request) def _recover_dbs(self, rc_data_path: str): + """ + CAUTION: last_block_info is not ready at this moment + + """ + + self._recover_commit(rc_data_path) + self._recover_rollback() + + def _recover_rollback(self): + Logger.debug(tag=ROLLBACK_LOG_TAG, msg=f"_recover_rollback() start") + + # Check if RollbackMetadata file exists + path = self._get_rollback_metadata_path() + if not os.path.isfile(path): + Logger.debug(tag=ROLLBACK_LOG_TAG, msg=f"_recover_rollback() end: {path} not found") + return + + try: + # Load RollbackMetadata from a file + with open(path, "rb") as f: + buf: bytes = f.read() + metadata = RollbackMetadata.from_bytes(buf) + except: + Logger.info(tag=ROLLBACK_LOG_TAG, msg=f"_recover_rollback() end: incomplete RollbackMetadata") + # os.remove(path) + return + + # Resume the previous rollback + context = self._context_factory.create(IconScoreContextType.DIRECT, block=metadata.last_block) + self._rollback(context, metadata.block_height, metadata.block_hash, metadata.term_start_block_height) + + # Clear backup files used for rollback (Optional) + + # Remove "ROLLBACK" file + # No "ROLLBACK" file means that there is no incomplete rollback + os.remove(path) + + Logger.debug(tag=WAL_LOG_TAG, msg=f"_recover_rollback() end") + + def _recover_commit(self, rc_data_path: str): """Recover iiss_db and state_db with a wal file :param rc_data_path: The directory where iiss_dbs are contained """ - Logger.debug(tag=WAL_LOG_TAG, msg=f"_recover_dbs() start: rc_data_path={rc_data_path}") + Logger.debug(tag=WAL_LOG_TAG, msg=f"_recover_commit() start: rc_data_path={rc_data_path}") path: str = self._get_write_ahead_log_path() if not os.path.isfile(path): - Logger.debug(tag=WAL_LOG_TAG, msg=f"_recover_dbs() end: No WAL file {path}") + Logger.debug(tag=WAL_LOG_TAG, msg=f"_recover_commit() end: No WAL file {path}") return self._wal_reader = None @@ -2173,7 +2229,7 @@ def _recover_dbs(self, rc_data_path: str): except BaseException as e: Logger.error(tag=WAL_LOG_TAG, msg=str(e)) - Logger.debug(tag=WAL_LOG_TAG, msg="_recover_dbs() end") + Logger.debug(tag=WAL_LOG_TAG, msg="_recover_commit() end") @staticmethod def _is_need_to_recover_rc_db(wal_state: 'WALState', is_calc_period_start_block: bool) -> bool: @@ -2249,6 +2305,9 @@ def _recover_state_db(self, reader: 'WriteAheadLogReader'): def _get_write_ahead_log_path(self) -> str: return os.path.join(self._state_db_root_path, self.WAL_FILE) + def _get_rollback_metadata_path(self) -> str: + return os.path.join(self._state_db_root_path, self.ROLLBACK_METADATA_FILE) + def hello(self) -> dict: """If state_db and rc_db are recovered, send COMMIT_BLOCK message to reward calculator It is called on INVOKE thread diff --git a/iconservice/rollback/metadata.py b/iconservice/rollback/metadata.py new file mode 100644 index 000000000..1a80a8b5b --- /dev/null +++ b/iconservice/rollback/metadata.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..base.block import Block +from ..icon_constant import Revision +from ..utils.msgpack_for_db import MsgPackForDB +from ..utils import bytes_to_hex + + +class Metadata(object): + """Contains rollback metadata + + """ + _VERSION = 0 + + def __init__(self, block_height: int, block_hash: bytes, term_start_block_height: int, last_block: 'Block'): + """ + + :param block_height: final block_height after rollback + :param block_hash: final block_hash after rollback + :param term_start_block_height: the start block height of the current term + :param last_block: the last block before rollback + """ + self._block_height = block_height + self._block_hash = block_hash + self._last_block = last_block + self._term_start_block_height = term_start_block_height + + @property + def block_height(self) -> int: + return self._block_height + + @property + def block_hash(self) -> bytes: + return self._block_hash + + @property + def last_block(self) -> 'Block': + return self._last_block + + @property + def term_start_block_height(self) -> int: + return self._term_start_block_height + + def __eq__(self, other): + return self._block_height == other.block_height \ + and self._block_hash == other.block_hash \ + and self._term_start_block_height == other.term_start_block_height \ + and self._last_block == other.last_block + + def __str__(self): + return f"RollbackMetadata: block_height={self._block_height} " \ + f"block_hash={bytes_to_hex(self._block_hash)} " \ + f"term_start_block_height={self._term_start_block_height} " \ + f"last_block={self._last_block}" + + @classmethod + def from_bytes(cls, buf: bytes) -> 'Metadata': + data: list = MsgPackForDB.loads(buf) + version: int = data[0] + assert version == cls._VERSION + + block_height: int = data[1] + block_hash: bytes = data[2] + term_start_block_height: int = data[3] + last_block: 'Block' = Block.from_bytes(data[4]) + + return Metadata(block_height, block_hash, term_start_block_height, last_block) + + def to_bytes(self) -> bytes: + data = [ + self._VERSION, + self._block_height, + self._block_hash, + self._term_start_block_height, + self._last_block.to_bytes(Revision.IISS.value) + ] + + return MsgPackForDB.dumps(data) diff --git a/iconservice/rollback/rollback_manager.py b/iconservice/rollback/rollback_manager.py index 53a24fe9f..fa49208cc 100644 --- a/iconservice/rollback/rollback_manager.py +++ b/iconservice/rollback/rollback_manager.py @@ -46,27 +46,28 @@ def __init__(self, backup_root_path: str, rc_data_path: str, state_db: 'KeyValue self._rc_data_path = rc_data_path self._state_db = state_db - def run(self, current_block_height: int, rollback_block_height: int, start_block_height_in_term: int): + def run(self, last_block_height: int, rollback_block_height: int, term_start_block_height: int): """Rollback to the previous block state - :param current_block_height: + :param last_block_height: the last confirmed block height :param rollback_block_height: the height of block to rollback to - :param start_block_height_in_term: + :param term_start_block_height: the start block height of the current term """ Logger.info(tag=TAG, msg=f"run() start: " - f"current_block_height={current_block_height} " - f"rollback_block_height={rollback_block_height}") + f"last_block_height={last_block_height} " + f"rollback_block_height={rollback_block_height} " + f"term_start_block_height={term_start_block_height}") - self._validate_block_heights(current_block_height, rollback_block_height) + self._validate_block_heights(last_block_height, rollback_block_height, term_start_block_height) - # Check whether a term change exists between current_block_height -1 and rollback_block_height, inclusive - term_change_exists = rollback_block_height < start_block_height_in_term - end_calc_block_height = start_block_height_in_term - 1 + term_change_exists = \ + self._term_change_exists(last_block_height, rollback_block_height, term_start_block_height) + calc_end_block_height = term_start_block_height - 1 reader = WriteAheadLogReader() state_db_batch = {} iiss_db_batch = {} - for block_height in range(current_block_height - 1, rollback_block_height - 1, -1): + for block_height in range(last_block_height - 1, rollback_block_height - 1, -1): # Make backup file with a given block_height path: str = self._get_backup_file_path(block_height) if not os.path.isfile(path): @@ -78,15 +79,15 @@ def run(self, current_block_height: int, rollback_block_height: int, start_block self._write_batch(reader.get_iterator(WALDBType.STATE.value), state_db_batch) # Merge backup data into iiss_db_batch - if not (term_change_exists and block_height > end_calc_block_height): + if not (term_change_exists and block_height > calc_end_block_height): self._write_batch(reader.get_iterator(WALDBType.RC.value), iiss_db_batch) reader.close() # If a term change is detected during rollback, handle the exceptions below if term_change_exists: - self._remove_block_produce_info(iiss_db_batch, end_calc_block_height) - self._rename_iiss_db(end_calc_block_height) + self._remove_block_produce_info(iiss_db_batch, calc_end_block_height) + self._rename_iiss_db(calc_end_block_height) # Commit write_batch to db self._commit_batch(state_db_batch, self._state_db) @@ -97,16 +98,23 @@ def run(self, current_block_height: int, rollback_block_height: int, start_block Logger.info(tag=TAG, msg="run() end") @staticmethod - def _validate_block_heights(current_block_height: int, rollback_block_height: int): - if current_block_height < 0: - raise InvalidParamsException(f"Invalid currentBlockHeight: {current_block_height}") + def _validate_block_heights(last_block_height: int, rollback_block_height: int, term_start_block_height: int): + if last_block_height < 0: + raise InvalidParamsException(f"Invalid lastBlockHeight: {last_block_height}") if rollback_block_height < 0: raise InvalidParamsException(f"Invalid rollbackBlockHeight: {rollback_block_height}") - if current_block_height <= rollback_block_height: + if term_start_block_height < 0: + raise InvalidParamsException(f"Invalid termStartBlockHeight: {term_start_block_height}") + + if rollback_block_height >= last_block_height: raise InvalidParamsException( - f"currentBlockHeight({current_block_height}) <= rollbackBlockHeight({rollback_block_height}") + f"lastBlockHeight({last_block_height}) <= rollbackBlockHeight({rollback_block_height}") + + @staticmethod + def _term_change_exists(last_block_height: int, rollback_block_height: int, term_start_block_height: int) -> bool: + return rollback_block_height < term_start_block_height <= last_block_height @staticmethod def _write_batch(it: Iterable[Tuple[bytes, Optional[bytes]]], batch: dict): diff --git a/tests/integrate_test/iiss/decentralized/test_rollback.py b/tests/integrate_test/iiss/decentralized/test_rollback.py index 72b6ecdbf..e71744a4a 100644 --- a/tests/integrate_test/iiss/decentralized/test_rollback.py +++ b/tests/integrate_test/iiss/decentralized/test_rollback.py @@ -77,6 +77,7 @@ class TestRollback(TestIISSBase): BLOCK_VALIDATION_PENALTY_THRESHOLD = 10 LOW_PRODUCTIVITY_PENALTY_THRESHOLD = 80 PENALTY_GRACE_PERIOD = CALCULATE_PERIOD * 2 + BLOCK_VALIDATION_PENALTY_THRESHOLD + BACKUP_FILES = CALCULATE_PERIOD def _make_init_config(self) -> dict: return { @@ -93,7 +94,8 @@ def _make_init_config(self) -> dict: ConfigKey.LOW_PRODUCTIVITY_PENALTY_THRESHOLD: self.LOW_PRODUCTIVITY_PENALTY_THRESHOLD, ConfigKey.PREP_MAIN_PREPS: self.MAIN_PREP_COUNT, ConfigKey.PREP_MAIN_AND_SUB_PREPS: self.ELECTED_PREP_COUNT, - ConfigKey.PENALTY_GRACE_PERIOD: self.PENALTY_GRACE_PERIOD + ConfigKey.PENALTY_GRACE_PERIOD: self.PENALTY_GRACE_PERIOD, + ConfigKey.BACKUP_FILES: self.BACKUP_FILES } def setUp(self): @@ -428,6 +430,58 @@ def test_rollback_multi_blocks(self): # Check the last block self._check_if_last_block_is_reverted(prev_block) + def test_rollback_with_term_change(self): + # Prevent icon_service_engine from sending RollbackRequest to rc + IconScoreContext.engine.iiss.rollback_reward_calculator = Mock() + + main_prep_count = PREP_MAIN_PREPS + elected_prep_count = PREP_MAIN_AND_SUB_PREPS + calculate_period = self.CALCULATE_PERIOD + + # Inspect the current term + term_2 = self.get_prep_term() + assert term_2["sequence"] == 2 + preps = term_2["preps"] + term_start_block_height = term_2["startBlockHeight"] + term_end_block_height = term_2["endBlockHeight"] + + _check_elected_prep_grades(preps, main_prep_count, elected_prep_count) + main_preps: List['Address'] = _get_main_preps(preps, main_prep_count) + + rollback_block: 'Block' = self.icon_service_engine._get_last_block() + assert rollback_block.height == term_start_block_height + + # Transfer 3000 icx to new 10 accounts + init_balance = icx_to_loop(3000) + accounts: List['EOAAccount'] = self.create_eoa_accounts(10) + self.distribute_icx(accounts=accounts, init_balance=init_balance) + + for account in accounts: + balance: int = self.get_balance(account.address) + assert balance == init_balance + + count = term_end_block_height - self.get_last_block().height + 1 + self.make_empty_blocks( + count=count, + prev_block_generator=main_preps[0], + prev_block_validators=[address for address in main_preps[1:]] + ) + + # TERM-3: Nothing + term_3: dict = self.get_prep_term() + assert term_3["sequence"] == 3 + assert term_3["startBlockHeight"] == self.get_last_block().height + + self._rollback(rollback_block) + + term: dict = self.get_prep_term() + assert term == term_2 + + # Check if the balances of accounts are reverted + for account in accounts: + balance: int = self.get_balance(account.address) + assert balance == 0 + def _rollback(self, block: 'Block'): super().rollback(block.height, block.hash) self._check_if_rollback_reward_calculator_is_called(block) diff --git a/tests/integrate_test/iiss/test_iiss_base.py b/tests/integrate_test/iiss/test_iiss_base.py index 6d20cd8ad..bbb53ca54 100644 --- a/tests/integrate_test/iiss/test_iiss_base.py +++ b/tests/integrate_test/iiss/test_iiss_base.py @@ -29,8 +29,9 @@ from tests.integrate_test.test_integrate_base import TestIntegrateBase, TOTAL_SUPPLY, DEFAULT_STEP_LIMIT if TYPE_CHECKING: - from tests.integrate_test.test_integrate_base import EOAAccount + from iconservice.base.block import Block from iconservice.iconscore.icon_score_result import TransactionResult + from tests.integrate_test.test_integrate_base import EOAAccount class TestIISSBase(TestIntegrateBase): @@ -586,5 +587,9 @@ def init_decentralized(self): tx_list.append(tx) self.process_confirm_block_tx(tx_list) - def get_debug_term(self) -> 'Term': - return IconScoreContext.engine.prep.term \ No newline at end of file + @staticmethod + def get_debug_term() -> 'Term': + return IconScoreContext.engine.prep.term + + def get_last_block(self) -> 'Block': + return self.icon_service_engine._get_last_block() diff --git a/tests/rollback/test_backup_manager.py b/tests/rollback/test_backup_manager.py index d02b61d5b..df2ccf24d 100644 --- a/tests/rollback/test_backup_manager.py +++ b/tests/rollback/test_backup_manager.py @@ -186,12 +186,12 @@ def _check_if_rollback_is_done(db: 'KeyValueDatabase', prev_state: OrderedDict): def _rollback_with_rollback_manager(self, last_block: 'Block'): rollback_manager = self.rollback_manager - current_block_height = last_block.height + 1 + last_block_height = last_block.height + 1 rollback_block_height = last_block.height # One block rollback ret = rollback_manager.run( - current_block_height, + last_block_height, rollback_block_height, - start_block_height_in_term=rollback_block_height - 1) + term_start_block_height=rollback_block_height - 1) assert ret is None diff --git a/tests/rollback/test_metadata.py b/tests/rollback/test_metadata.py new file mode 100644 index 000000000..a2951be41 --- /dev/null +++ b/tests/rollback/test_metadata.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest +import os +import time +from iconservice.rollback.metadata import Metadata +from iconservice.base.block import Block + + +class TestMetadata(unittest.TestCase): + def setUp(self) -> None: + block_height: int = random.randint(1000, 10000) + block_hash: bytes = os.urandom(32) + prev_block_hash: bytes = os.urandom(32) + timestamp_us: int = int(time.time() * 1000_000) + cumulative_fee = random.randint(0, 10000) + + last_block = Block( + block_height=block_height, + block_hash=block_hash, + timestamp=timestamp_us, + prev_hash=prev_block_hash, + cumulative_fee=cumulative_fee + ) + + block_height = last_block.height - 10 + block_hash: bytes = os.urandom(32) + term_start_block_height = block_height = block_height - 20 + last_block: 'Block' = last_block + + metadata = Metadata(block_height, block_hash, term_start_block_height, last_block) + assert metadata.block_height == block_height + assert metadata.block_hash == block_hash + assert metadata.term_start_block_height == term_start_block_height + assert metadata.last_block == last_block + + self.metadata = metadata + self.block_height = block_height + self.block_hash = block_hash + self.term_start_block_height = term_start_block_height + self.last_block = last_block + + def test_from_bytes(self): + metadata = self.metadata + + buf: bytes = metadata.to_bytes() + assert isinstance(buf, bytes) + + metadata2 = Metadata.from_bytes(buf) + assert metadata2 == metadata + assert id(metadata2) != id(metadata) + assert metadata2.block_height == self.block_height + assert metadata2.block_hash == self.block_hash + assert metadata2.term_start_block_height == self.term_start_block_height + assert metadata2.last_block == self.last_block From 6c328a454e66fcc14ba31c1234a660bf14ebcb5b Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Thu, 19 Dec 2019 20:58:54 +0900 Subject: [PATCH 27/36] Remove ROLLBACK_METADATA file after rollback is done --- iconservice/icon_service_engine.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 23f692fd5..9a1f871fb 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -2037,6 +2037,8 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: context = self._context_factory.create(IconScoreContextType.DIRECT, block=last_block) self._rollback(context, block_height, block_hash, term_start_block_height) + self._remove_rollback_metadata() + except BaseException as e: Logger.error(tag=ROLLBACK_LOG_TAG, msg=str(e)) raise InternalServiceErrorException( @@ -2176,19 +2178,19 @@ def _recover_rollback(self): buf: bytes = f.read() metadata = RollbackMetadata.from_bytes(buf) except: - Logger.info(tag=ROLLBACK_LOG_TAG, msg=f"_recover_rollback() end: incomplete RollbackMetadata") - # os.remove(path) - return + Logger.info(tag=ROLLBACK_LOG_TAG, msg="_recover_rollback() end: incomplete RollbackMetadata") + metadata = None - # Resume the previous rollback - context = self._context_factory.create(IconScoreContextType.DIRECT, block=metadata.last_block) - self._rollback(context, metadata.block_height, metadata.block_hash, metadata.term_start_block_height) + if metadata: + # Resume the previous rollback + context = self._context_factory.create(IconScoreContextType.DIRECT, block=metadata.last_block) + self._rollback(context, metadata.block_height, metadata.block_hash, metadata.term_start_block_height) - # Clear backup files used for rollback (Optional) + # Clear backup files used for rollback (Optional) # Remove "ROLLBACK" file # No "ROLLBACK" file means that there is no incomplete rollback - os.remove(path) + self._remove_rollback_metadata() Logger.debug(tag=WAL_LOG_TAG, msg=f"_recover_rollback() end") @@ -2308,6 +2310,18 @@ def _get_write_ahead_log_path(self) -> str: def _get_rollback_metadata_path(self) -> str: return os.path.join(self._state_db_root_path, self.ROLLBACK_METADATA_FILE) + def _remove_rollback_metadata(self): + Logger.debug(tag=ROLLBACK_LOG_TAG, msg="_remove_rollback_metadata() start") + + try: + path = self._get_rollback_metadata_path() + os.remove(path) + Logger.info(tag=ROLLBACK_LOG_TAG, msg=f"Remove {path}") + except: + Logger.error(tag=ROLLBACK_LOG_TAG, msg=f"Failed to remove {path}") + + Logger.debug(tag=ROLLBACK_LOG_TAG, msg="_remove_rollback_metadata() end") + def hello(self) -> dict: """If state_db and rc_db are recovered, send COMMIT_BLOCK message to reward calculator It is called on INVOKE thread From 91808b8ced678b7a90be55b09bb8fbadb5e0e180 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Fri, 20 Dec 2019 16:23:30 +0900 Subject: [PATCH 28/36] Remove trailing "_" from rc_db_name prefixes in RewardCalcStorage --- iconservice/iiss/reward_calc/storage.py | 14 +++++++++++--- iconservice/rollback/rollback_manager.py | 16 ++++++++++------ .../iiss/decentralized/test_rc_db_data.py | 4 ++-- .../iiss/test_recover_using_wal.py | 2 +- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/iconservice/iiss/reward_calc/storage.py b/iconservice/iiss/reward_calc/storage.py index ec6a8a58b..646e72631 100644 --- a/iconservice/iiss/reward_calc/storage.py +++ b/iconservice/iiss/reward_calc/storage.py @@ -67,8 +67,8 @@ class Storage(object): """ CURRENT_IISS_DB_NAME = "current_db" - STANDBY_IISS_DB_NAME_PREFIX = "standby_rc_db_" - IISS_RC_DB_NAME_PREFIX = "iiss_rc_db_" + STANDBY_IISS_DB_NAME_PREFIX = "standby_rc_db" + IISS_RC_DB_NAME_PREFIX = "iiss_rc_db" KEY_FOR_GETTING_LAST_TRANSACTION_INDEX = b'last_transaction_index' KEY_FOR_CALC_RESPONSE_FROM_RC = b'calc_response_from_rc' @@ -144,7 +144,15 @@ def _supplement_db(self, context: 'IconScoreContext', revision: int): @classmethod def get_standby_rc_db_name(cls, block_height: int) -> str: - return f"{cls.STANDBY_IISS_DB_NAME_PREFIX}{block_height}" + return cls._get_db_name(cls.STANDBY_IISS_DB_NAME_PREFIX, block_height) + + @classmethod + def get_iiss_rc_db_name(cls, block_height: int) -> str: + return cls._get_db_name(cls.IISS_RC_DB_NAME_PREFIX, block_height) + + @classmethod + def _get_db_name(cls, prefix: str, block_height: int) -> str: + return f"{prefix}_{block_height}" def put_data_directly(self, iiss_data: 'Data', tx_index: Optional[int] = None): if isinstance(iiss_data, TxData): diff --git a/iconservice/rollback/rollback_manager.py b/iconservice/rollback/rollback_manager.py index fa49208cc..5d15f2a2f 100644 --- a/iconservice/rollback/rollback_manager.py +++ b/iconservice/rollback/rollback_manager.py @@ -87,7 +87,7 @@ def run(self, last_block_height: int, rollback_block_height: int, term_start_blo # If a term change is detected during rollback, handle the exceptions below if term_change_exists: self._remove_block_produce_info(iiss_db_batch, calc_end_block_height) - self._rename_iiss_db(calc_end_block_height) + self._rename_iiss_db_to_current_db(calc_end_block_height) # Commit write_batch to db self._commit_batch(state_db_batch, self._state_db) @@ -145,11 +145,15 @@ def _remove_backup_file(cls, path: str): except BaseException as e: Logger.debug(tag=TAG, msg=str(e)) - def _rename_iiss_db(self, end_calc_block_height: int): - Logger.debug(tag=TAG, msg=f"_rename_iiss_db() start: end_calc_block_height={end_calc_block_height}") + def _rename_iiss_db_to_current_db(self, calc_end_block_height: int): + """Rename iiss_db to current_db - src_path = os.path.join( - self._rc_data_path, f"{RewardCalcStorage.IISS_RC_DB_NAME_PREFIX}{end_calc_block_height}") + """ + Logger.debug(tag=TAG, + msg=f"_rename_iiss_db_to_current_db() start: calc_end_block_height={calc_end_block_height}") + + filename = RewardCalcStorage.get_iiss_rc_db_name(calc_end_block_height) + src_path = os.path.join(self._rc_data_path, filename) dst_path = os.path.join(self._rc_data_path, RewardCalcStorage.CURRENT_IISS_DB_NAME) Logger.info(tag=TAG, msg=f"rename_iiss_db: src_path={src_path} dst_path={dst_path}") @@ -158,7 +162,7 @@ def _rename_iiss_db(self, end_calc_block_height: int): # Rename iiss_rc_db_{BH} to current_db os.rename(src_path, dst_path) - Logger.debug(tag=TAG, msg="_rename_iiss_db() end") + Logger.debug(tag=TAG, msg="_rename_iiss_db_to_current_db() end") @classmethod def _remove_block_produce_info(cls, iiss_db_batch: dict, block_height: int): diff --git a/tests/integrate_test/iiss/decentralized/test_rc_db_data.py b/tests/integrate_test/iiss/decentralized/test_rc_db_data.py index 370852857..72ec7e561 100644 --- a/tests/integrate_test/iiss/decentralized/test_rc_db_data.py +++ b/tests/integrate_test/iiss/decentralized/test_rc_db_data.py @@ -40,11 +40,11 @@ def setUp(self): def get_last_rc_db_data(rc_data_path): return sorted([dir_name for dir_name in os.listdir(rc_data_path) if dir_name.startswith(RewardCalcStorage.IISS_RC_DB_NAME_PREFIX)], - key=lambda rc_dir: int(rc_dir[len(RewardCalcStorage.IISS_RC_DB_NAME_PREFIX):]), + key=lambda rc_dir: int(rc_dir[len(RewardCalcStorage.IISS_RC_DB_NAME_PREFIX)+1:]), reverse=True)[0] def _check_the_name_of_rc_db(self, actual_rc_db_name: str): - expected_last_rc_db_name: str = Storage.IISS_RC_DB_NAME_PREFIX + str(self._block_height - 1) + expected_last_rc_db_name: str = f"{Storage.IISS_RC_DB_NAME_PREFIX}_{self._block_height - 1}" self.assertEqual(expected_last_rc_db_name, actual_rc_db_name) def test_all_rc_db_data_block_height(self): diff --git a/tests/integrate_test/iiss/test_recover_using_wal.py b/tests/integrate_test/iiss/test_recover_using_wal.py index 73d7e248e..9aee255b3 100644 --- a/tests/integrate_test/iiss/test_recover_using_wal.py +++ b/tests/integrate_test/iiss/test_recover_using_wal.py @@ -170,7 +170,7 @@ def _check_the_state_and_rc_db_after_recover(self, block_height: int, is_calc_pe self.assertEqual(RCDataCheckFlag.ALL_ON_START, rc_data_flag) self.assertTrue( os.path.isdir( - os.path.join(rc_data_path, f"{RewardCalcStorage.IISS_RC_DB_NAME_PREFIX}{block_height - 1}") + os.path.join(rc_data_path, f"{RewardCalcStorage.IISS_RC_DB_NAME_PREFIX}_{block_height - 1}") ) ) else: From 9741bda612649bc82c9b84ac400fa3772cb5de72 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Fri, 20 Dec 2019 18:58:07 +0900 Subject: [PATCH 29/36] Refactor WAL recovery * Refactor IconServiceEngine._recover_rc_db() * Remove RewardCalcStorage.scan_rc_db() * Optimize code --- iconservice/icon_service_engine.py | 74 ++++++++++++++----------- iconservice/iiss/reward_calc/storage.py | 28 ---------- 2 files changed, 41 insertions(+), 61 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 9a1f871fb..a3cede6ac 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -15,6 +15,7 @@ import os from copy import deepcopy +from enum import IntEnum from typing import TYPE_CHECKING, List, Any, Optional, Tuple, Dict, Union from iconcommons.logger import Logger @@ -34,7 +35,7 @@ from .base.transaction import Transaction from .base.type_converter_templates import ConstantKeys from .database.factory import ContextDatabaseFactory -from .database.wal import WriteAheadLogReader +from .database.wal import WriteAheadLogReader, WALDBType from .database.wal import WriteAheadLogWriter, IissWAL, StateWAL, WALState from .deploy import DeployEngine, DeployStorage from .deploy.icon_builtin_score_loader import IconBuiltinScoreLoader @@ -68,11 +69,11 @@ from .precommit_data_manager import PrecommitData, PrecommitDataManager, PrecommitFlag from .prep import PRepEngine, PRepStorage from .prep.data import PRep +from .rollback.metadata import Metadata as RollbackMetadata from .utils import print_log_with_level from .utils import sha3_256, int_to_bytes, ContextEngine, ContextStorage from .utils import to_camel_case, bytes_to_hex from .utils.bloom import BloomFilter -from .rollback.metadata import Metadata as RollbackMetadata if TYPE_CHECKING: from .iconscore.icon_score_event_log import EventLog @@ -2256,41 +2257,48 @@ def _recover_rc_db(cls, reader: 'WriteAheadLogReader', rc_data_path: str): # If WAL file is made at the start block of calc period if is_calc_period_start_block: - current_rc_db_path, standby_rc_db_path, iiss_rc_db_path = RewardCalcStorage.scan_rc_db(rc_data_path) - is_current_exists: bool = len(current_rc_db_path) > 0 - is_standby_exists: bool = len(standby_rc_db_path) > 0 - is_iiss_exists: bool = len(iiss_rc_db_path) > 0 - Logger.info(tag=WAL_LOG_TAG, - msg=f"current_exists={is_current_exists}, " - f"is_standby_exists={is_standby_exists}, " - f"is_iiss_exists={is_iiss_exists}") - - # If only current_db exists, replace current_db to standby_rc_db - if is_current_exists and not is_standby_exists and not is_iiss_exists: - # Process compaction before send the RC DB to reward calculator - prev_calc_db_path: str = os.path.join(rc_data_path, RewardCalcStorage.CURRENT_IISS_DB_NAME) - RewardCalcStorage.process_db_compaction(prev_calc_db_path) - - calculate_block_height: int = reader.block.height - 1 - standby_rc_db_path: str = \ - RewardCalcStorage.rename_current_db_to_standby_db(rc_data_path, calculate_block_height) - is_standby_exists: bool = True - elif not is_current_exists and not is_standby_exists: - # No matter iiss_db exists or not, If both current_db and standby_db do not exist, raise error - raise DatabaseException(f"RC related DB not exists") - - current_db: 'KeyValueDatabase' = RewardCalcStorage.create_current_db(rc_data_path) - if is_standby_exists: - RewardCalcStorage.rename_standby_db_to_iiss_db(standby_rc_db_path) - else: - current_db: 'KeyValueDatabase' = RewardCalcStorage.create_current_db(rc_data_path) + cls._recover_rc_db_on_calc_start_block(rc_data_path, reader.block) # Write data to "current_db" - current_db.write_batch(reader.get_iterator(0)) + current_db: 'KeyValueDatabase' = RewardCalcStorage.create_current_db(rc_data_path) + current_db.write_batch(reader.get_iterator(WALDBType.RC.value)) current_db.close() Logger.debug(tag=WAL_LOG_TAG, msg="_recover_rc_db() end") + @classmethod + def _recover_rc_db_on_calc_start_block(cls, rc_data_path: str, block: 'Block'): + class DBType(IntEnum): + CURRENT = 0 + STANDBY = 1 + IISS = 2 + + calc_end_block_height = block.height - 1 + + items = ( + [RewardCalcStorage.CURRENT_IISS_DB_NAME, False], + [RewardCalcStorage.get_standby_rc_db_name(calc_end_block_height), False], + [RewardCalcStorage.get_iiss_rc_db_name(calc_end_block_height), False] + ) + + for item in items: + item[0]: str = os.path.join(rc_data_path, item[0]) + item[1]: bool = os.path.isdir(item[0]) + + if items[DBType.IISS][1]: + # Do nothing + pass + elif items[DBType.STANDBY][1]: + # Rename standby_rc_db to iiss_rc_db + os.rename(items[DBType.STANDBY][0], items[DBType.IISS][0]) + elif items[DBType.CURRENT][1]: + # Compact current_db and rename current_db to iiss_rc_db + path: str = items[DBType.CURRENT][0] + RewardCalcStorage.process_db_compaction(path) + os.rename(path, items[DBType.IISS][0]) + else: + raise DatabaseException("IISS-related DB not found") + def _recover_state_db(self, reader: 'WriteAheadLogReader'): Logger.debug(tag=WAL_LOG_TAG, msg=f"_recover_state_db() start: reader={reader}") @@ -2299,7 +2307,7 @@ def _recover_state_db(self, reader: 'WriteAheadLogReader'): Logger.info(tag=WAL_LOG_TAG, msg="state_db has already been up-to-date") return - ret: int = self._icx_context_db.key_value_db.write_batch(reader.get_iterator(1)) + ret: int = self._icx_context_db.key_value_db.write_batch(reader.get_iterator(WALDBType.STATE.value)) Logger.info(tag=WAL_LOG_TAG, msg=f"state_db has been updated with wal file: count={ret}") Logger.debug(tag=WAL_LOG_TAG, msg="_recover_state_db() end") @@ -2313,8 +2321,8 @@ def _get_rollback_metadata_path(self) -> str: def _remove_rollback_metadata(self): Logger.debug(tag=ROLLBACK_LOG_TAG, msg="_remove_rollback_metadata() start") + path = self._get_rollback_metadata_path() try: - path = self._get_rollback_metadata_path() os.remove(path) Logger.info(tag=ROLLBACK_LOG_TAG, msg=f"Remove {path}") except: diff --git a/iconservice/iiss/reward_calc/storage.py b/iconservice/iiss/reward_calc/storage.py index 646e72631..a34ca5572 100644 --- a/iconservice/iiss/reward_calc/storage.py +++ b/iconservice/iiss/reward_calc/storage.py @@ -295,34 +295,6 @@ def rename_standby_db_to_iiss_db(cls, standby_db_path: str) -> str: return iiss_db_path - @classmethod - def scan_rc_db(cls, rc_data_path: str) -> Tuple[str, str, str]: - """Scan directories that are managed by RewardCalcStorage - - :param rc_data_path: the parent directory of rc_dbs - :return: current_rc_db_exists(bool), standby_rc_db_path, iiss_rc_db_path - """ - current_rc_db_path: str = "" - standby_rc_db_path: str = "" - iiss_rc_db_path: str = "" - - with os.scandir(rc_data_path) as it: - for entry in it: - if entry.is_dir(): - if entry.name == cls.CURRENT_IISS_DB_NAME: - current_rc_db_path: str = os.path.join(rc_data_path, cls.CURRENT_IISS_DB_NAME) - elif entry.name.startswith(cls.STANDBY_IISS_DB_NAME_PREFIX): - standby_rc_db_path: str = os.path.join(rc_data_path, entry.name) - elif entry.name.startswith(cls.IISS_RC_DB_NAME_PREFIX): - iiss_rc_db_path: str = os.path.join(rc_data_path, entry.name) - - Logger.info(tag=WAL_LOG_TAG, - msg=f"current_rc_db={current_rc_db_path}, " - f"standby_rc_db={standby_rc_db_path}, " - f"iiss_rc_db={iiss_rc_db_path}") - - return current_rc_db_path, standby_rc_db_path, iiss_rc_db_path - def get_total_elected_prep_delegated_snapshot(self) -> int: """ total_elected_prep_delegated_snapshot = From e5aa5380633ad4af4c845708c4c2b59bacbd3d3b Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Mon, 23 Dec 2019 17:34:08 +0900 Subject: [PATCH 30/36] Implement to recover rollback * Add load() and save() to rollback.Metadata class * Add _recover_rollback() and _finish_to_recover_rollback() to IconServiceEngine. --- iconservice/icon_service_engine.py | 103 +++++++++++++++++------------ iconservice/rollback/metadata.py | 33 ++++++++- tests/rollback/test_metadata.py | 21 +++++- 3 files changed, 110 insertions(+), 47 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index a3cede6ac..6fb18384a 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -2029,10 +2029,9 @@ def rollback(self, block_height: int, block_hash: bytes) -> dict: # Record rollback metadata path = self._get_rollback_metadata_path() - with open(path, "wb") as f: - metadata = RollbackMetadata( - block_height, block_hash, term_start_block_height, last_block) - f.write(metadata.to_bytes()) + metadata = RollbackMetadata( + block_height, block_hash, term_start_block_height, last_block) + metadata.save(path) # Do rollback context = self._context_factory.create(IconScoreContextType.DIRECT, block=last_block) @@ -2084,9 +2083,6 @@ def _rollback(self, context: 'IconScoreContext', # Close storage IconScoreContext.storage.rc.close() - # Rollback the state of reward_calculator prior to iconservice - IconScoreContext.engine.iiss.rollback_reward_calculator(rollback_block_height, rollback_block_hash) - # Rollback state_db and rc_data_db to those of a given block_height rollback_manager = RollbackManager( self._backup_root_path, self._rc_data_path, self._icx_context_db.key_value_db) @@ -2095,6 +2091,9 @@ def _rollback(self, context: 'IconScoreContext', rollback_block_height=rollback_block_height, term_start_block_height=term_start_block_height) + # Rollback the state of reward_calculator prior to iconservice + IconScoreContext.engine.iiss.rollback_reward_calculator(rollback_block_height, rollback_block_hash) + # Clear all iconscores and reload builtin scores only builtin_score_owner: 'Address' = Address.from_string(self._conf[ConfigKey.BUILTIN_SCORE_OWNER]) self._load_builtin_scores(context, builtin_score_owner) @@ -2126,6 +2125,7 @@ def _rollback(self, context: 'IconScoreContext', for engine in engines: engine.rollback(context, rollback_block_height, rollback_block_hash) + # Reset last_block self._init_last_block_info(context) def clear_context_stack(self): @@ -2167,31 +2167,18 @@ def _recover_dbs(self, rc_data_path: str): def _recover_rollback(self): Logger.debug(tag=ROLLBACK_LOG_TAG, msg=f"_recover_rollback() start") - # Check if RollbackMetadata file exists + # Load RollbackMetadata from a file path = self._get_rollback_metadata_path() - if not os.path.isfile(path): - Logger.debug(tag=ROLLBACK_LOG_TAG, msg=f"_recover_rollback() end: {path} not found") - return - - try: - # Load RollbackMetadata from a file - with open(path, "rb") as f: - buf: bytes = f.read() - metadata = RollbackMetadata.from_bytes(buf) - except: - Logger.info(tag=ROLLBACK_LOG_TAG, msg="_recover_rollback() end: incomplete RollbackMetadata") - metadata = None + metadata: Optional['RollbackMetadata'] = RollbackMetadata.load(path) if metadata: - # Resume the previous rollback - context = self._context_factory.create(IconScoreContextType.DIRECT, block=metadata.last_block) - self._rollback(context, metadata.block_height, metadata.block_hash, metadata.term_start_block_height) - - # Clear backup files used for rollback (Optional) - - # Remove "ROLLBACK" file - # No "ROLLBACK" file means that there is no incomplete rollback - self._remove_rollback_metadata() + # Resume the previous rollback for the databases managed by iconservice + rollback_manager = RollbackManager( + self._backup_root_path, self._rc_data_path, self._icx_context_db.key_value_db) + rollback_manager.run( + last_block_height=metadata.last_block.height, + rollback_block_height=metadata.block_height, + term_start_block_height=metadata.term_start_block_height) Logger.debug(tag=WAL_LOG_TAG, msg=f"_recover_rollback() end") @@ -2339,26 +2326,56 @@ def hello(self) -> dict: """ Logger.debug(tag=self.TAG, msg="hello() start") - iiss_engine: 'IISSEngine' = IconScoreContext.engine.iiss + self._finish_to_recover_commit() + self._finish_to_recover_rollback() + + Logger.debug(tag=self.TAG, msg="hello() end") + return {} + + def _finish_to_recover_commit(self): + """Finish to recover WAL by sending COMMIT_BLOCK message to reward calculator + """ + Logger.debug(tag=self.TAG, msg="_finish_to_recover_commit() start") + + if not isinstance(self._wal_reader, WriteAheadLogReader): + Logger.debug(tag=self.TAG, msg="_finish_to_recover_commit() end") + return + + iiss_engine: 'IISSEngine' = IconScoreContext.engine.iiss last_block: 'Block' = self._get_last_block() - if isinstance(self._wal_reader, WriteAheadLogReader): - wal_state = WALState(self._wal_reader.state) + wal_state = WALState(self._wal_reader.state) - # If only writing rc_db is done on commit without sending COMMIT_BLOCK to rc, - # send COMMIT_BLOCK to rc prior to invoking a block - if not (wal_state & WALState.SEND_COMMIT_BLOCK): - iiss_engine.send_commit( - self._wal_reader.block.height, self._wal_reader.instant_block_hash) + # If only writing rc_db is done on commit without sending COMMIT_BLOCK to rc, + # send COMMIT_BLOCK to rc prior to invoking a block + if not (wal_state & WALState.SEND_COMMIT_BLOCK): + iiss_engine.send_commit( + self._wal_reader.block.height, self._wal_reader.instant_block_hash) - assert last_block == self._wal_reader.block + assert last_block == self._wal_reader.block - # No need to use - self._wal_reader = None + # No need to use + self._wal_reader = None - # iiss_engine.init_reward_calculator(last_block) + Logger.debug(tag=self.TAG, msg="_finish_to_recover_commit() end") - Logger.debug(tag=self.TAG, msg="hello() end") + def _finish_to_recover_rollback(self): + """Finish to recover rollback by sending ROLLBACK message to reward calculator + """ + Logger.debug(tag=self.TAG, msg="_finish_to_recover_rollback() start") - return {} + # Get ROLLBACK_METADATA file path + path = self._get_rollback_metadata_path() + + # Load RollbackMetadata from a file + metadata: Optional[RollbackMetadata] = RollbackMetadata.load(path) + + if metadata: + # Request reward calculator to rollback its state + IconScoreContext.engine.iiss.rollback_reward_calculator(metadata.block_height, metadata.block_hash) + + # Remove ROLLBACK_METADATA file when the rollback is done. + self._remove_rollback_metadata() + + Logger.debug(tag=self.TAG, msg="_finish_to_recover_rollback() end") diff --git a/iconservice/rollback/metadata.py b/iconservice/rollback/metadata.py index 1a80a8b5b..c80d64c99 100644 --- a/iconservice/rollback/metadata.py +++ b/iconservice/rollback/metadata.py @@ -13,10 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +__all__ = "Metadata" + +from typing import Optional + +from iconcommons.logger import Logger + from ..base.block import Block +from ..icon_constant import ROLLBACK_LOG_TAG from ..icon_constant import Revision -from ..utils.msgpack_for_db import MsgPackForDB from ..utils import bytes_to_hex +from ..utils.msgpack_for_db import MsgPackForDB + +TAG = ROLLBACK_LOG_TAG class Metadata(object): @@ -89,3 +98,25 @@ def to_bytes(self) -> bytes: ] return MsgPackForDB.dumps(data) + + @classmethod + def load(cls, path: str) -> Optional['Metadata']: + Logger.debug(tag=TAG, msg=f"from_path() start: {path}") + + metadata = None + + try: + with open(path, "rb") as f: + buf: bytes = f.read() + metadata = Metadata.from_bytes(buf) + except FileNotFoundError: + Logger.debug(tag=TAG, msg=f"File not found: {path}") + except BaseException as e: + Logger.info(tag=TAG, msg=f"Unexpected error: {str(e)}") + + Logger.debug(tag=TAG, msg=f"from_path() end: metadata={metadata}") + return metadata + + def save(self, path: str): + with open(path, "wb") as f: + f.write(self.to_bytes()) diff --git a/tests/rollback/test_metadata.py b/tests/rollback/test_metadata.py index a2951be41..2e3fae2a6 100644 --- a/tests/rollback/test_metadata.py +++ b/tests/rollback/test_metadata.py @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import random -import unittest import os +import random import time -from iconservice.rollback.metadata import Metadata +import unittest +from typing import Optional + from iconservice.base.block import Block +from iconservice.rollback.metadata import Metadata class TestMetadata(unittest.TestCase): @@ -67,3 +69,16 @@ def test_from_bytes(self): assert metadata2.block_hash == self.block_hash assert metadata2.term_start_block_height == self.term_start_block_height assert metadata2.last_block == self.last_block + + def test_load(self): + path = "./ROLLBACK_METADATA" + metadata: Optional['Metadata'] = Metadata.load(path) + assert metadata is None + + self.metadata.save(path) + + metadata = Metadata.load(path) + assert metadata == self.metadata + assert id(metadata) != id(self.metadata) + + os.remove(path) From f35f03d171c4ce7197a3128291646b3b315763f5 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Mon, 23 Dec 2019 19:09:03 +0900 Subject: [PATCH 31/36] Correct some logging message typos on rollback.Metadata --- iconservice/rollback/metadata.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/iconservice/rollback/metadata.py b/iconservice/rollback/metadata.py index c80d64c99..e5af69de2 100644 --- a/iconservice/rollback/metadata.py +++ b/iconservice/rollback/metadata.py @@ -70,10 +70,11 @@ def __eq__(self, other): and self._last_block == other.last_block def __str__(self): - return f"RollbackMetadata: block_height={self._block_height} " \ + return f"rollback.Metadata(" \ + f"block_height={self._block_height} " \ f"block_hash={bytes_to_hex(self._block_hash)} " \ f"term_start_block_height={self._term_start_block_height} " \ - f"last_block={self._last_block}" + f"last_block={self._last_block})" @classmethod def from_bytes(cls, buf: bytes) -> 'Metadata': @@ -101,7 +102,7 @@ def to_bytes(self) -> bytes: @classmethod def load(cls, path: str) -> Optional['Metadata']: - Logger.debug(tag=TAG, msg=f"from_path() start: {path}") + Logger.debug(tag=TAG, msg=f"load() start: {path}") metadata = None @@ -114,9 +115,13 @@ def load(cls, path: str) -> Optional['Metadata']: except BaseException as e: Logger.info(tag=TAG, msg=f"Unexpected error: {str(e)}") - Logger.debug(tag=TAG, msg=f"from_path() end: metadata={metadata}") + Logger.debug(tag=TAG, msg=f"load() end: metadata={metadata}") return metadata def save(self, path: str): + Logger.debug(tag=TAG, msg=f"save() start: {path}") + with open(path, "wb") as f: f.write(self.to_bytes()) + + Logger.debug(tag=TAG, msg=f"save() end") From 7fbec9b9067f5c0d963213e384fd68afa1e81ca2 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 24 Dec 2019 12:08:27 +0900 Subject: [PATCH 32/36] Make rollback state rocovery robust * Consider the case that renaming iiss_db to current_db has been already done --- iconservice/rollback/rollback_manager.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/iconservice/rollback/rollback_manager.py b/iconservice/rollback/rollback_manager.py index 5d15f2a2f..f904ac0ff 100644 --- a/iconservice/rollback/rollback_manager.py +++ b/iconservice/rollback/rollback_manager.py @@ -157,10 +157,12 @@ def _rename_iiss_db_to_current_db(self, calc_end_block_height: int): dst_path = os.path.join(self._rc_data_path, RewardCalcStorage.CURRENT_IISS_DB_NAME) Logger.info(tag=TAG, msg=f"rename_iiss_db: src_path={src_path} dst_path={dst_path}") - # Remove a new current_db - shutil.rmtree(dst_path, ignore_errors=True) - # Rename iiss_rc_db_{BH} to current_db - os.rename(src_path, dst_path) + # Consider the case that renaming iiss_db to current_db has been already done + if os.path.isdir(src_path): + # Remove a new current_db + shutil.rmtree(dst_path, ignore_errors=True) + # Rename iiss_rc_db_{BH} to current_db + os.rename(src_path, dst_path) Logger.debug(tag=TAG, msg="_rename_iiss_db_to_current_db() end") From 8fcc9d82352a96f5a50fd2c5c83a7bf788f80749 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Tue, 24 Dec 2019 18:21:58 +0900 Subject: [PATCH 33/36] Change old-fashioned iiss_db_name * iiss_rc_db_{BH}_{revision} -> iiss_rc_db_{BH} * Add IissDBNameRefactor class --- iconservice/icon_service_engine.py | 4 + iconservice/iiss/reward_calc/storage.py | 55 ++++++++- tests/iiss/test_iiss_db_name_refactor.py | 150 +++++++++++++++++++++++ 3 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 tests/iiss/test_iiss_db_name_refactor.py diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 6fb18384a..5a1bbf49e 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -63,6 +63,7 @@ from .icx.issue.base_transaction_creator import BaseTransactionCreator from .iiss import IISSEngine, IISSStorage, check_decentralization_condition from .iiss.reward_calc import RewardCalcStorage, RewardCalcDataCreator +from .iiss.reward_calc.storage import IissDBNameRefactor from .iiss.reward_calc.storage import RewardCalcDBInfo from .inner_call import inner_call from .meta import MetaDBStorage @@ -183,6 +184,9 @@ def open(self, conf: 'IconConfig'): context = IconScoreContext(IconScoreContextType.DIRECT) self._init_last_block_info(context) + # Remove revision from iiss_rc_db name + IissDBNameRefactor.run(self._rc_data_path) + # Clean up stale backup files self._backup_cleaner.run(context.block.height) diff --git a/iconservice/iiss/reward_calc/storage.py b/iconservice/iiss/reward_calc/storage.py index a34ca5572..d223995f3 100644 --- a/iconservice/iiss/reward_calc/storage.py +++ b/iconservice/iiss/reward_calc/storage.py @@ -25,7 +25,7 @@ from ...database.db import KeyValueDatabase from ...icon_constant import ( DATA_BYTE_ORDER, Revision, RC_DATA_VERSION_TABLE, RC_DB_VERSION_0, - IISS_LOG_TAG, WAL_LOG_TAG, ROLLBACK_LOG_TAG + IISS_LOG_TAG, ROLLBACK_LOG_TAG ) from ...iconscore.icon_score_context import IconScoreContext from ...iiss.reward_calc.data_creator import DataCreator @@ -329,3 +329,56 @@ def get_total_elected_prep_delegated_snapshot(self) -> int: msg=f"get_total_elected_prep_delegated_snapshot load: {ret}") return ret + + +class IissDBNameRefactor(object): + """Change iiss_db name: remove revision from iiss_db name + + """ + _DB_NAME_PREFIX = Storage.IISS_RC_DB_NAME_PREFIX + + @classmethod + def run(cls, rc_data_path: str) -> int: + ret = 0 + + with os.scandir(rc_data_path) as it: + for entry in it: + if entry.is_dir() and entry.name.startswith(cls._DB_NAME_PREFIX): + new_name: str = cls._get_db_name_without_revision(entry.name) + if not new_name: + Logger.info( + tag=IISS_LOG_TAG, + msg=f"Refactoring iiss_db name has been already done: old={entry.name} " + f"rc_data_path={rc_data_path}") + break + + cls._change_db_name(rc_data_path, entry.name, new_name) + ret += 1 + + return ret + + @classmethod + def _change_db_name(cls, rc_data_path: str, old_name: str, new_name: str): + if old_name == new_name: + return + + src_path: str = os.path.join(rc_data_path, old_name) + dst_path: str = os.path.join(rc_data_path, new_name) + + try: + os.rename(src_path, dst_path) + Logger.info(tag=IISS_LOG_TAG, msg=f"Renaming iiss_db_name succeeded: old={old_name} new={new_name}") + except BaseException as e: + Logger.error(tag=IISS_LOG_TAG, + msg=f"Failed to rename iiss_db_name: old={old_name} new={new_name} " + f"path={rc_data_path} exception={str(e)}") + + @classmethod + def _get_db_name_without_revision(cls, name: str) -> Optional[str]: + # items[0]: block_height, items[1]: revision + items: List[str] = name[len(cls._DB_NAME_PREFIX) + 1:].split("_") + if len(items) == 1: + # No need to rename + return None + + return f"{cls._DB_NAME_PREFIX}_{items[0]}" diff --git a/tests/iiss/test_iiss_db_name_refactor.py b/tests/iiss/test_iiss_db_name_refactor.py new file mode 100644 index 000000000..1d98a0595 --- /dev/null +++ b/tests/iiss/test_iiss_db_name_refactor.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import random +import shutil +import unittest +from typing import Optional + +from iconservice.iiss.reward_calc.storage import IissDBNameRefactor + + +class TestIissDBNameRefactor(unittest.TestCase): + RC_DATA_PATH = "./for_iiss_db_name_refactor_test" + + def setUp(self) -> None: + os.mkdir(self.RC_DATA_PATH) + + revision = random.randint(0, 10) + block_height = random.randint(100, 10000000000) + + self.rc_data_path = self.RC_DATA_PATH + self.new_name = self._get_new_db_name(block_height) + self.old_name = self._get_old_db_name(block_height, revision) + + def tearDown(self) -> None: + shutil.rmtree(self.RC_DATA_PATH, ignore_errors=True) + + def test_run(self): + term_period = 43120 + rc_data_path = self.rc_data_path + start_block_height = 12_690_545 + revision = 0 + size = 5 + + # Make the directories whose name is old-fashioned + for i in range(size): + block_height = start_block_height + term_period * i + + old_name = self._get_old_db_name(block_height, revision) + path = os.path.join(rc_data_path, old_name) + os.mkdir(path) + + ret = IissDBNameRefactor.run(rc_data_path) + assert ret == size + + # Check whether old-fashioned iiss_db names are changed to the new one + for i in range(size): + block_height = start_block_height + term_period * i + + new_name = self._get_new_db_name(block_height) + path = os.path.join(rc_data_path, new_name) + assert os.path.isdir(path) + + def test_run_with_already_renamed_db(self): + term_period = 43120 + rc_data_path = self.rc_data_path + start_block_height = 12_690_545 + size = 5 + + # Make the directories whose name is old-fashioned + for i in range(size): + block_height = start_block_height + term_period * i + + name = self._get_new_db_name(block_height) + path = os.path.join(rc_data_path, name) + os.mkdir(path) + + # Make a new-fashioned db + new_name = self._get_new_db_name(start_block_height + term_period * size) + path = os.path.join(rc_data_path, new_name) + os.mkdir(path) + + ret = IissDBNameRefactor.run(rc_data_path) + assert ret == 0 + + # Check whether old-fashioned iiss_db names are changed to the new one + for i in range(size): + block_height = start_block_height + term_period * i + + name = self._get_new_db_name(block_height) + path = os.path.join(rc_data_path, name) + assert os.path.isdir(path) + + def test__get_db_name_without_revision(self): + new_name = self.new_name + old_name = self.old_name + + name: Optional[str] = IissDBNameRefactor._get_db_name_without_revision(new_name) + assert name is None + + name: Optional[str] = IissDBNameRefactor._get_db_name_without_revision(old_name) + assert name == new_name + + def test__change_db_name_success(self): + rc_data_path = self.rc_data_path + old_name = self.old_name + new_name = self.new_name + + old_path = os.path.join(rc_data_path, old_name) + new_path = os.path.join(rc_data_path, new_name) + + os.mkdir(old_path) + assert os.path.isdir(old_path) + + # Success case + IissDBNameRefactor._change_db_name(rc_data_path, old_name, new_name) + assert not os.path.exists(old_path) + assert os.path.isdir(new_path) + + def test__change_db_name_failure(self): + rc_data_path = self.rc_data_path + old_name = self.old_name + new_name = self.new_name + + old_path = os.path.join(rc_data_path, old_name) + new_path = os.path.join(rc_data_path, new_name) + + os.mkdir(old_path) + assert os.path.isdir(old_path) + + # Failure case 1: rename non-existent dir to new_name + IissDBNameRefactor._change_db_name(rc_data_path, "no_dir", new_name) + assert os.path.isdir(old_path) + assert not os.path.exists(new_path) + + # Failure case 2: rename a dir to the same name + IissDBNameRefactor._change_db_name(rc_data_path, old_name, old_name) + assert os.path.isdir(old_path) + assert not os.path.exists(new_path) + + @staticmethod + def _get_old_db_name(block_height: int, revision: int) -> str: + return f"{IissDBNameRefactor._DB_NAME_PREFIX}_{block_height}_{revision}" + + @staticmethod + def _get_new_db_name(block_height: int) -> str: + return f"{IissDBNameRefactor._DB_NAME_PREFIX}_{block_height}" From e2638c23c91b3f7b7b9a3caff5e62618cb10f3a9 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Thu, 2 Jan 2020 10:58:35 +0900 Subject: [PATCH 34/36] Change block backup filename format * block-12345.bak -> block-0000012345.bak --- iconservice/rollback/__init__.py | 2 +- tests/rollback/test_backup_manager.py | 8 ++++--- tests/rollback/test_functions.py | 33 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 tests/rollback/test_functions.py diff --git a/iconservice/rollback/__init__.py b/iconservice/rollback/__init__.py index 6dded5800..c55f16d43 100644 --- a/iconservice/rollback/__init__.py +++ b/iconservice/rollback/__init__.py @@ -25,7 +25,7 @@ def get_backup_filename(block_height: int) -> str: :param block_height: the height of the block where we want to go back :return: """ - return f"block-{block_height}.bak" + return f"block-{block_height:010d}.bak" def check_backup_exists(backup_root_path: str, current_block_height: int, rollback_block_height: int) -> bool: diff --git a/tests/rollback/test_backup_manager.py b/tests/rollback/test_backup_manager.py index df2ccf24d..c3b61fd01 100644 --- a/tests/rollback/test_backup_manager.py +++ b/tests/rollback/test_backup_manager.py @@ -20,12 +20,13 @@ from collections import OrderedDict from iconservice.base.block import Block -from iconservice.rollback.backup_manager import BackupManager from iconservice.database.db import KeyValueDatabase -from iconservice.rollback.rollback_manager import RollbackManager from iconservice.database.wal import WriteAheadLogReader, WALDBType from iconservice.icon_constant import Revision from iconservice.iiss.reward_calc.storage import Storage as RewardCalcStorage +from iconservice.rollback import get_backup_filename +from iconservice.rollback.backup_manager import BackupManager +from iconservice.rollback.rollback_manager import RollbackManager def _create_dummy_data(count: int) -> OrderedDict: @@ -165,7 +166,8 @@ def _commit_rc_db(db: 'KeyValueDatabase', rc_batch: OrderedDict): assert len(rc_batch) == count def _rollback(self, last_block: 'Block'): - backup_path = os.path.join(self.backup_root_path, f"block-{last_block.height}.bak") + filename: str = get_backup_filename(last_block.height) + backup_path = os.path.join(self.backup_root_path, filename) reader = WriteAheadLogReader() reader.open(backup_path) diff --git a/tests/rollback/test_functions.py b/tests/rollback/test_functions.py new file mode 100644 index 000000000..8fc16ca8e --- /dev/null +++ b/tests/rollback/test_functions.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 ICON Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest +from iconservice.rollback import get_backup_filename + + +class TestFunctions(unittest.TestCase): + def test_get_backup_filename(self): + heights = [0, 1, 123456789, 1234567890] + expected_filenames = [ + "block-0000000000.bak", + "block-0000000001.bak", + "block-0123456789.bak", + "block-1234567890.bak", + ] + + for i in range(len(heights)): + filename: str = get_backup_filename(heights[i]) + assert filename == expected_filenames[i] From 346432db4f3414ae2e5c849481baba282a9cf7a8 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Thu, 2 Jan 2020 17:14:00 +0900 Subject: [PATCH 35/36] Update BackupCleaner * Divide run() into run_on_init(), run_on_commit() and run() * Update the unittest for BackupCleaner --- iconservice/icon_service_engine.py | 11 +- iconservice/rollback/__init__.py | 2 +- iconservice/rollback/backup_cleaner.py | 109 +++++++++++-- tests/rollback/test_backup_cleaner.py | 214 +++++++++++++++++++++---- tests/rollback/test_functions.py | 8 +- 5 files changed, 290 insertions(+), 54 deletions(-) diff --git a/iconservice/icon_service_engine.py b/iconservice/icon_service_engine.py index 5a1bbf49e..821546124 100644 --- a/iconservice/icon_service_engine.py +++ b/iconservice/icon_service_engine.py @@ -188,7 +188,7 @@ def open(self, conf: 'IconConfig'): IissDBNameRefactor.run(self._rc_data_path) # Clean up stale backup files - self._backup_cleaner.run(context.block.height) + self._backup_cleaner.run_on_init(context.block.height) # set revision (if governance SCORE does not exist, remain revision to default). try: @@ -1860,8 +1860,8 @@ def _commit_after_iiss(self, is_calc_period_start_block=is_calc_period_start_block, instant_block_hash=instant_block_hash) - # Clean up stale backup files - self._backup_cleaner.run(context.block.height) + # Clean up the oldest backup file + self._backup_cleaner.run_on_commit(context.block.height) # Write iiss_wal to rc_db standby_db_info: Optional['RewardCalcDBInfo'] = \ @@ -2382,4 +2382,9 @@ def _finish_to_recover_rollback(self): # Remove ROLLBACK_METADATA file when the rollback is done. self._remove_rollback_metadata() + # Remove obsolete block backup files used for rollback + self._backup_cleaner.run( + start_block_height=metadata.block_height + 1, + end_block_height=metadata.last_block.height - 1) + Logger.debug(tag=self.TAG, msg="_finish_to_recover_rollback() end") diff --git a/iconservice/rollback/__init__.py b/iconservice/rollback/__init__.py index c55f16d43..d03ec27cf 100644 --- a/iconservice/rollback/__init__.py +++ b/iconservice/rollback/__init__.py @@ -25,7 +25,7 @@ def get_backup_filename(block_height: int) -> str: :param block_height: the height of the block where we want to go back :return: """ - return f"block-{block_height:010d}.bak" + return f"{block_height:010d}.bak" def check_backup_exists(backup_root_path: str, current_block_height: int, rollback_block_height: int) -> bool: diff --git a/iconservice/rollback/backup_cleaner.py b/iconservice/rollback/backup_cleaner.py index cc7959546..256dabd38 100644 --- a/iconservice/rollback/backup_cleaner.py +++ b/iconservice/rollback/backup_cleaner.py @@ -16,6 +16,7 @@ __all__ = "BackupCleaner" import os +import re from iconcommons.logger import Logger @@ -37,32 +38,110 @@ def __init__(self, backup_root_path: str, backup_files: int): :param backup_files: the maximum backup files to keep """ self._backup_root_path = backup_root_path - self._backup_files = backup_files if backup_files >= 0 else BACKUP_FILES + self._backup_files = backup_files if backup_files > 0 else BACKUP_FILES + self._regex_object = re.compile("^[\d]{10}.bak$") - def run(self, current_block_height: int) -> int: - """Clean up old backup files + def run_on_init(self, current_block_height: int) -> int: + """Clean up all stale backup files on iconservice startup + + :param current_block_height: + :return: + """ + Logger.debug(tag=_TAG, msg=f"run_on_init() start") + + ret = 0 + start_block_height = max(0, current_block_height - self._backup_files) + + with os.scandir(self._backup_root_path) as it: + for entry in it: + # backup filename: ex) 0000012345.bak + if entry.is_file() and self._is_backup_filename_valid(entry.name): + block_height: int = self._get_block_height_from_filename(entry.name) + if block_height < 0: + continue + + # Do nothing for the latest backup files + if start_block_height <= block_height < current_block_height: + continue + + # Remove stale backup files + if self._remove_file(entry.path): + ret += 1 + + Logger.debug(tag=_TAG, msg=f"run_on_init() end: ret={ret}") + return ret + + @staticmethod + def _get_block_height_from_filename(filename: str) -> int: + try: + return int(filename[:-4]) + except: + pass + + return -1 + + def _is_backup_filename_valid(self, filename: str) -> bool: + return len(filename) == 14 and bool(self._regex_object.match(filename)) + + def run_on_commit(self, current_block_height: int) -> int: + """Remove the oldest backup file on commit :param: current_block_height :param: func: function to remove a file with path :return: the number of removed files """ - Logger.debug(tag=_TAG, msg=f"run() start: current_block_height={current_block_height}") + Logger.debug(tag=_TAG, + msg=f"run() start: current_block_height={current_block_height} " + f"backup_files={self._backup_files}") + + # Remove the oldest backup file only + start_block_height = current_block_height - self._backup_files - 1 + ret = self.run(start_block_height, end_block_height=start_block_height) + + Logger.debug(tag=_TAG, msg="run() end") + + return ret + + def run(self, start_block_height: int, end_block_height: int) -> int: + """Remove block backup files ranging from start_block_height to end_block_height inclusive + + :param start_block_height: + :param end_block_height: + :return: The number of removed files + """ + Logger.debug(tag=_TAG, msg=f"run() start: start={start_block_height} end={end_block_height}") + + # Parameters sanity check + if start_block_height < 0 or start_block_height > end_block_height: + Logger.warning(tag=_TAG, msg=f"Invalid range: start={start_block_height} end={end_block_height}") + return -1 ret = 0 - start = current_block_height - self._backup_files - 1 - try: - for block_height in range(start, -1, -1): - filename: str = get_backup_filename(block_height) - path: str = os.path.join(self._backup_root_path, filename) - os.remove(path) + # Remove block backup files ranging from start_block_height to end_block_height inclusive + for block_height in range(start_block_height, end_block_height + 1): + filename: str = get_backup_filename(block_height) + path: str = os.path.join(self._backup_root_path, filename) + + # Remove a file ignoring any exceptions + if self._remove_file(path): ret += 1 + + Logger.info(tag=_TAG, + msg=f"Clean up old backup files: " + f"start={start_block_height} end={end_block_height} count={ret}") + Logger.debug(tag=_TAG, msg=f"run() end: ret={ret}") + + return ret + + @staticmethod + def _remove_file(path: str) -> bool: + try: + os.remove(path) + return True except FileNotFoundError: pass except BaseException as e: - Logger.debug(tag=_TAG, msg=str(e)) - - Logger.info(tag=_TAG, msg=f"Clean up old backup files: start={start} count={ret}") - Logger.debug(tag=_TAG, msg="run() end") + Logger.warning(tag=_TAG, msg=str(e)) - return ret + return False diff --git a/tests/rollback/test_backup_cleaner.py b/tests/rollback/test_backup_cleaner.py index 7e919712e..0593d8425 100644 --- a/tests/rollback/test_backup_cleaner.py +++ b/tests/rollback/test_backup_cleaner.py @@ -13,34 +13,39 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import os +import random import shutil +import unittest -from iconservice.rollback.backup_cleaner import BackupCleaner -from iconservice.rollback import get_backup_filename from iconservice.icon_constant import BACKUP_FILES +from iconservice.rollback import get_backup_filename +from iconservice.rollback.backup_cleaner import BackupCleaner + +def _get_backup_file_path(backup_root_path: str, block_height: int) -> str: + filename = get_backup_filename(block_height) + return os.path.join(backup_root_path, filename) -def _create_dummy_backup_files(backup_root_path: str, current_block_height: int, backup_files: int): - for i in range(backup_files): - block_height = current_block_height - i - 1 + +def _create_dummy_backup_files(backup_root_path: str, start_block_height: int, end_block_height: int): + for block_height in range(start_block_height, end_block_height + 1): if block_height < 0: break - filename = get_backup_filename(block_height) - path: str = os.path.join(backup_root_path, filename) - open(path, "w").close() + path: str = _get_backup_file_path(backup_root_path, block_height) + _create_dummy_file(path) - assert os.path.isfile(path) +def _create_dummy_file(path: str): + open(path, "w").close() + assert os.path.isfile(path) -def _check_if_backup_files_exists(backup_root_path: str, start_block_height, count, expected: bool): - for i in range(count): - block_height = start_block_height + i - filename = get_backup_filename(block_height) - path = os.path.join(backup_root_path, filename) +def _check_if_backup_files_exists( + backup_root_path: str, start_block_height: int, end_block_height: int, expected: bool): + for block_height in range(start_block_height, end_block_height + 1): + path: str = _get_backup_file_path(backup_root_path, block_height) assert os.path.exists(path) == expected @@ -48,6 +53,8 @@ class TestBackupCleaner(unittest.TestCase): def setUp(self) -> None: backup_files = 10 backup_root_path = os.path.join(os.path.dirname(__file__), "backup") + + shutil.rmtree(backup_root_path, ignore_errors=True) os.mkdir(backup_root_path) backup_cleaner = BackupCleaner(backup_root_path, backup_files) @@ -57,48 +64,193 @@ def setUp(self) -> None: self.backup_cleaner = backup_cleaner def tearDown(self) -> None: - shutil.rmtree(self.backup_root_path) + shutil.rmtree(self.backup_root_path, ignore_errors=True) def test__init__(self): backup_cleaner = BackupCleaner("./haha", backup_files=-10) assert backup_cleaner._backup_files >= 0 assert backup_cleaner._backup_files == BACKUP_FILES - def test_run_with_too_many_backup_files(self): + def test__get_block_height_from_filename(self): + filenames = ["0123456789.bak", "tmp.bak", "12345.bak", "ff.bak", "011.bak"] + expected_block_heights = [123456789, -1, 12345, -1, 11] + + for i in range(len(filenames)): + block_height = BackupCleaner._get_block_height_from_filename(filenames[i]) + assert block_height == expected_block_heights[i] + + def test__is_backup_filename_valid(self): + backup_root_path: str = self.backup_root_path + backup_cleaner = BackupCleaner(backup_root_path, backup_files=10) + + filenames = ["0000000001.bak", "0123456789.bak", "tmp.bak", "12345.bak", "ff.bak", "000000001f.bak"] + expected_results = [True, True, False, False, False, False] + + for i in range(len(filenames)): + result: bool = backup_cleaner._is_backup_filename_valid(filenames[i]) + assert result == expected_results[i] + + def test_run_on_commit_with_too_many_backup_files(self): current_block_height = 101 - dummy_backup_files = 20 backup_files = 10 backup_root_path: str = self.backup_root_path backup_cleaner = BackupCleaner(backup_root_path, backup_files) # Create 20 dummy backup files: block-81.bak ... block-100.back - _create_dummy_backup_files(backup_root_path, current_block_height, dummy_backup_files) + _create_dummy_backup_files(backup_root_path, 81, 100) - # Remove block-81.bak ~ block-90.bak - backup_cleaner.run(current_block_height) + # Remove 0000000090.bak file only + ret = backup_cleaner.run_on_commit(current_block_height) + assert ret == 1 - # Check if too old backup files are removed - _check_if_backup_files_exists(backup_root_path, 81, 10, expected=False) + # Check if 0000000090.bak is removed + _check_if_backup_files_exists(backup_root_path, 90, 90, expected=False) + + _check_if_backup_files_exists(backup_root_path, 81, 89, expected=True) # Check if the latest backup files exist (block-91.bak ~ block-100.bak) - _check_if_backup_files_exists(backup_root_path, 91, 10, expected=True) + _check_if_backup_files_exists(backup_root_path, 91, 100, expected=True) - def test_run_with_too_short_backup_files(self): + def test_run_on_commit_with_too_short_backup_files(self): current_block_height = 101 - dummy_backup_files = 5 backup_files = 10 backup_root_path: str = self.backup_root_path backup_cleaner = BackupCleaner(backup_root_path, backup_files) # Create 5 dummy backup files: block-96.bak ... block-100.back - _create_dummy_backup_files(backup_root_path, current_block_height, dummy_backup_files) + _create_dummy_backup_files(backup_root_path, 96, 100) # No backup file will be removed - backup_cleaner.run(current_block_height) + ret = backup_cleaner.run_on_commit(current_block_height) + assert ret == 0 + + # Check if the latest backup files exist (block-96.bak ~ block-100.bak) + _check_if_backup_files_exists(backup_root_path, 96, 100, expected=True) + + def test_run(self): + backup_files = 10 + backup_root_path: str = self.backup_root_path + backup_cleaner = BackupCleaner(backup_root_path, backup_files) + + # Create 100 dummy backup files: 101 ~ 200 + start_block_height = 101 + end_block_height = 200 + count = end_block_height - start_block_height + 1 + _create_dummy_backup_files(backup_root_path, start_block_height, end_block_height) + + # Remove all dummy backup files above: 101 ~ 200 + ret = backup_cleaner.run(start_block_height, end_block_height) + assert ret == count + + # Check if the latest backup files exist (block-96.bak ~ block-100.bak) + _check_if_backup_files_exists(backup_root_path, start_block_height, end_block_height, expected=False) + + def test_run_with_some_dropped_files(self): + backup_files = 10 + backup_root_path: str = self.backup_root_path + backup_cleaner = BackupCleaner(backup_root_path, backup_files) + + # Create 100 dummy backup files: 101 ~ 200 + start_block_height = 101 + end_block_height = 200 + count = end_block_height - start_block_height + 1 + _create_dummy_backup_files(backup_root_path, start_block_height, end_block_height) + + # Choose 5 block_heights randomly and remove them for test + # Although block_heights are overlapped by accident, no problem + dropped_block_heights = set() + for _ in range(5): + block_height = random.randint(start_block_height, end_block_height) + dropped_block_heights.add(block_height) + + assert 0 < len(dropped_block_heights) <= 5 + + for block_height in dropped_block_heights: + path = _get_backup_file_path(backup_root_path, block_height) + try: + os.remove(path) + except: + pass + + # Remove all dummy backup files above: 101 ~ 200 + ret = backup_cleaner.run(start_block_height, end_block_height) + assert ret == count - len(dropped_block_heights) # Check if the latest backup files exist (block-96.bak ~ block-100.bak) - _check_if_backup_files_exists(backup_root_path, 96, 5, expected=True) + _check_if_backup_files_exists(backup_root_path, start_block_height, end_block_height, expected=False) + + def test_run_sanity_check(self): + backup_files = 10 + backup_root_path: str = self.backup_root_path + backup_cleaner = BackupCleaner(backup_root_path, backup_files) + + # Case 1: start_block_height < 0 + ret = backup_cleaner.run(start_block_height=-20, end_block_height=100) + assert ret < 0 + + # Case 2: end_block_height < 0 + ret = backup_cleaner.run(start_block_height=0, end_block_height=-1) + assert ret < 0 + + # Case 3: start_block_height > end_block_height + ret = backup_cleaner.run(start_block_height=10, end_block_height=9) + assert ret < 0 + + # Case 4: start_block_height == end_block_height + start_block_height = 10 + end_block_height = 10 + count = 1 + _create_dummy_backup_files(backup_root_path, start_block_height, end_block_height) + + ret = backup_cleaner.run(start_block_height, end_block_height) + assert ret == count + + _check_if_backup_files_exists(backup_root_path, start_block_height, end_block_height, expected=False) + + def test_run_on_init(self): + current_block_height = 101 + backup_files = 10 + backup_root_path: str = self.backup_root_path + backup_cleaner = BackupCleaner(backup_root_path, backup_files) + # Create 100 dummy backup files: 0 ~ 100 + _create_dummy_backup_files(backup_root_path, 0, 100) + + # Remove all stale backup files except for the latest ones: 91 ~ 100 + ret = backup_cleaner.run_on_init(current_block_height) + assert ret == 91 + + # Check if too old backup files are removed + _check_if_backup_files_exists(backup_root_path, 0, 90, expected=False) + + # Check if the latest backup files exist (block-91.bak ~ block-100.bak) + _check_if_backup_files_exists(backup_root_path, 91, 100, expected=True) + + def test_run_on_init_with_invalid_files(self): + current_block_height = 101 + backup_files = 10 + backup_root_path: str = self.backup_root_path + backup_cleaner = BackupCleaner(backup_root_path, backup_files) + + # Create 100 dummy backup files: 0 ~ 100 + _create_dummy_backup_files(backup_root_path, 81, 100) + + # Create invalid files + filenames = ["10.bak", "tmp", "011.bak", "tmp123.bak", "000000001f.bak", "000000001F.bak"] + for filename in filenames: + path = os.path.join(backup_root_path, filename) + _create_dummy_file(path) + + # Remove all stale backup files except for the latest ones: 91 ~ 100 + ret = backup_cleaner.run_on_init(current_block_height) + assert ret == 10 + + # Check if too old backup files are removed + _check_if_backup_files_exists(backup_root_path, 81, 90, expected=False) + + # Check if the latest backup files exist (block-91.bak ~ block-100.bak) + _check_if_backup_files_exists(backup_root_path, 91, 100, expected=True) -if __name__ == '__main__': - unittest.main() + for filename in filenames: + path = os.path.join(backup_root_path, filename) + assert os.path.isfile(path) diff --git a/tests/rollback/test_functions.py b/tests/rollback/test_functions.py index 8fc16ca8e..e80e3158b 100644 --- a/tests/rollback/test_functions.py +++ b/tests/rollback/test_functions.py @@ -22,10 +22,10 @@ class TestFunctions(unittest.TestCase): def test_get_backup_filename(self): heights = [0, 1, 123456789, 1234567890] expected_filenames = [ - "block-0000000000.bak", - "block-0000000001.bak", - "block-0123456789.bak", - "block-1234567890.bak", + "0000000000.bak", + "0000000001.bak", + "0123456789.bak", + "1234567890.bak", ] for i in range(len(heights)): From 3056ce776a1ee10d527656f43ee49481e5e05d33 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Fri, 3 Jan 2020 15:01:20 +0900 Subject: [PATCH 36/36] VERSION: 1.6.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 83d4cf8a9..dc1e644a1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.18 +1.6.0