From d3812635041f25c7e4617da55d3f76ce78b99a68 Mon Sep 17 00:00:00 2001 From: leeheonseung Date: Tue, 10 Mar 2020 19:09:55 +0900 Subject: [PATCH 1/2] IS-1029: Array Optimization hotfix * cache prefix_hash_key --- iconservice/database/db.py | 28 +- iconservice/iconscore/icon_container_db.py | 94 +++--- .../test_integrate_container_db_patch.py | 2 +- tests/test_container_db.py | 284 ++++++++++++++++++ 4 files changed, 350 insertions(+), 58 deletions(-) create mode 100644 tests/test_container_db.py diff --git a/iconservice/database/db.py b/iconservice/database/db.py index 3a430eed8..0ef427a4f 100644 --- a/iconservice/database/db.py +++ b/iconservice/database/db.py @@ -384,6 +384,14 @@ def __init__(self, self._context_db = context_db self._observer: Optional[DatabaseObserver] = None + self.prefix_hash_key: bytes = self._make_prefix_hash_key() + + def _make_prefix_hash_key(self) -> bytes: + data = [self.address.to_bytes()] + if self._prefix is not None: + data.append(self._prefix) + return b'|'.join(data) + b'|' + def get(self, key: bytes) -> bytes: """ Gets the value for the specified key @@ -460,12 +468,8 @@ def _hash_key(self, key: bytes) -> bytes: :params key: key passed by SCORE :return: key bytes """ - data = [self.address.to_bytes()] - if self._prefix is not None: - data.append(self._prefix) - data.append(key) - return b'|'.join(data) + return b''.join((self.prefix_hash_key, key)) def _validate_ownership(self): """Prevent a SCORE from accessing the database of another SCORE @@ -490,6 +494,14 @@ def __init__(self, address: 'Address', score_db: 'IconScoreDatabase', prefix: by self._prefix = prefix self._score_db = score_db + self.prefix_hash_key: bytes = self._make_prefix_hash_key() + + def _make_prefix_hash_key(self) -> bytes: + data = [] + if self._prefix is not None: + data.append(self._prefix) + return b'|'.join(data) + b'|' + def get(self, key: bytes) -> bytes: """ Gets the value for the specified key @@ -544,9 +556,5 @@ def _hash_key(self, key: bytes) -> bytes: :params key: key passed by SCORE :return: key bytes """ - data = [] - if self._prefix is not None: - data.append(self._prefix) - data.append(key) - return b'|'.join(data) + return b''.join((self.prefix_hash_key, key)) diff --git a/iconservice/iconscore/icon_container_db.py b/iconservice/iconscore/icon_container_db.py index f226cd0c7..46310b261 100644 --- a/iconservice/iconscore/icon_container_db.py +++ b/iconservice/iconscore/icon_container_db.py @@ -16,10 +16,10 @@ from typing import TypeVar, Optional, Any, Union, TYPE_CHECKING -from .icon_score_context import ContextContainer +from iconservice.icon_constant import IconScoreContextType, Revision +from iconservice.iconscore.icon_score_context import ContextContainer from ..base.address import Address from ..base.exception import InvalidParamsException, InvalidContainerAccessException -from ..icon_constant import Revision, IconScoreContextType from ..utils import int_to_bytes, bytes_to_int if TYPE_CHECKING: @@ -39,27 +39,27 @@ def get_encoded_key(key: V) -> bytes: class ContainerUtil(object): - @staticmethod - def create_db_prefix(cls, var_key: K) -> bytes: + @classmethod + def create_db_prefix(cls, container_cls: type, var_key: K) -> bytes: """Create a prefix used as a parameter of IconScoreDatabase.get_sub_db() - :param cls: ArrayDB, DictDB, VarDB + :param container_cls: ArrayDB, DictDB, VarDB :param var_key: :return: """ - if cls == ArrayDB: + if container_cls == ArrayDB: container_id = ARRAY_DB_ID - elif cls == DictDB: + elif container_cls == DictDB: container_id = DICT_DB_ID else: - raise InvalidParamsException(f'Unsupported container class: {cls}') + raise InvalidParamsException(f'Unsupported container class: {container_cls}') encoded_key: bytes = get_encoded_key(var_key) return b'|'.join([container_id, encoded_key]) - @staticmethod - def encode_key(key: K) -> bytes: + @classmethod + def encode_key(cls, key: K) -> bytes: """Create a key passed to IconScoreDatabase :param key: @@ -80,8 +80,8 @@ def encode_key(key: K) -> bytes: raise InvalidParamsException(f'Unsupported key type: {type(key)}') return bytes_key - @staticmethod - def encode_value(value: V) -> bytes: + @classmethod + def encode_value(cls, value: V) -> bytes: if isinstance(value, int): byte_value = int_to_bytes(value) elif isinstance(value, str): @@ -96,8 +96,8 @@ def encode_value(value: V) -> bytes: raise InvalidParamsException(f'Unsupported value type: {type(value)}') return byte_value - @staticmethod - def decode_object(value: bytes, value_type: type) -> Optional[Union[K, V]]: + @classmethod + def decode_object(cls, value: bytes, value_type: type) -> Optional[Union[K, V]]: if value is None: return get_default_value(value_type) @@ -114,45 +114,45 @@ def decode_object(value: bytes, value_type: type) -> Optional[Union[K, V]]: obj_value = value return obj_value - @staticmethod - def remove_prefix_from_iters(iter_items: iter) -> iter: - return ((ContainerUtil.__remove_prefix_from_key(key), value) for key, value in iter_items) + @classmethod + def remove_prefix_from_iters(cls, iter_items: iter) -> iter: + return ((cls.__remove_prefix_from_key(key), value) for key, value in iter_items) - @staticmethod - def __remove_prefix_from_key(key_from_bytes: bytes) -> bytes: + @classmethod + def __remove_prefix_from_key(cls, key_from_bytes: bytes) -> bytes: return key_from_bytes[:-1] - @staticmethod - def put_to_db(db: 'IconScoreDatabase', db_key: str, container: iter) -> None: - sub_db = db.get_sub_db(ContainerUtil.encode_key(db_key)) + @classmethod + def put_to_db(cls, db: 'IconScoreDatabase', db_key: str, container: iter) -> None: + sub_db = db.get_sub_db(cls.encode_key(db_key)) if isinstance(container, dict): - ContainerUtil.__put_to_db_internal(sub_db, container.items()) + cls.__put_to_db_internal(sub_db, container.items()) elif isinstance(container, (list, set, tuple)): - ContainerUtil.__put_to_db_internal(sub_db, enumerate(container)) + cls.__put_to_db_internal(sub_db, enumerate(container)) - @staticmethod - def get_from_db(db: 'IconScoreDatabase', db_key: str, *args, value_type: type) -> Optional[K]: - sub_db = db.get_sub_db(ContainerUtil.encode_key(db_key)) + @classmethod + def get_from_db(cls, db: 'IconScoreDatabase', db_key: str, *args, value_type: type) -> Optional[K]: + sub_db = db.get_sub_db(cls.encode_key(db_key)) *args, last_arg = args for arg in args: - sub_db = sub_db.get_sub_db(ContainerUtil.encode_key(arg)) + sub_db = sub_db.get_sub_db(cls.encode_key(arg)) - byte_key = sub_db.get(ContainerUtil.encode_key(last_arg)) + byte_key = sub_db.get(cls.encode_key(last_arg)) if byte_key is None: return get_default_value(value_type) - return ContainerUtil.decode_object(byte_key, value_type) + return cls.decode_object(byte_key, value_type) - @staticmethod - def __put_to_db_internal(db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], iters: iter) -> None: + @classmethod + def __put_to_db_internal(cls, db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], iters: iter) -> None: for key, value in iters: - sub_db = db.get_sub_db(ContainerUtil.encode_key(key)) + sub_db = db.get_sub_db(cls.encode_key(key)) if isinstance(value, dict): - ContainerUtil.__put_to_db_internal(sub_db, value.items()) + cls.__put_to_db_internal(sub_db, value.items()) elif isinstance(value, (list, set, tuple)): - ContainerUtil.__put_to_db_internal(sub_db, enumerate(value)) + cls.__put_to_db_internal(sub_db, enumerate(value)) else: - db_key = ContainerUtil.encode_key(key) - db_value = ContainerUtil.encode_value(value) + db_key = cls.encode_key(key) + db_value = cls.encode_value(value) db.put(db_key, db_value) @@ -279,12 +279,12 @@ def __get_size(self) -> int: return self.__get_size_from_db() def __get_size_from_db(self) -> int: - return ContainerUtil.decode_object(self._db.get(ArrayDB.__SIZE_BYTE_KEY), int) + return ContainerUtil.decode_object(self._db.get(self.__SIZE_BYTE_KEY), int) def __set_size(self, size: int) -> None: self.__legacy_size = size byte_value = ContainerUtil.encode_value(size) - self._db.put(ArrayDB.__SIZE_BYTE_KEY, byte_value) + self._db.put(self.__SIZE_BYTE_KEY, byte_value) def __put(self, index: int, value: V) -> None: byte_value = ContainerUtil.encode_value(value) @@ -312,7 +312,7 @@ def __setitem__(self, index: int, value: V) -> None: raise InvalidParamsException('ArrayDB out of index') def __getitem__(self, index: int) -> V: - return ArrayDB._get(self._db, self.__get_size(), index, self.__value_type) + return self._get(self._db, self.__get_size(), index, self.__value_type) def __contains__(self, item: V): for e in self: @@ -320,14 +320,14 @@ def __contains__(self, item: V): return True return False - @staticmethod - def __is_defective_revision(): + @classmethod + def __is_defective_revision(cls): context = ContextContainer._get_context() revision = context.revision return context.type == IconScoreContextType.INVOKE and revision < Revision.THREE.value - @staticmethod - def _get(db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, index: int, value_type: type) -> V: + @classmethod + def _get(cls, db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, index: int, value_type: type) -> V: if not isinstance(index, int): raise InvalidParamsException('Invalid index type: not an integer') @@ -341,10 +341,10 @@ def _get(db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, inde raise InvalidParamsException('ArrayDB out of index') - @staticmethod - def _get_generator(db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, value_type: type): + @classmethod + def _get_generator(cls, db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, value_type: type): for index in range(size): - yield ArrayDB._get(db, size, index, value_type) + yield cls._get(db, size, index, value_type) class VarDB(object): diff --git a/tests/integrate_test/test_integrate_container_db_patch.py b/tests/integrate_test/test_integrate_container_db_patch.py index c958138e9..d3b8a5c03 100644 --- a/tests/integrate_test/test_integrate_container_db_patch.py +++ b/tests/integrate_test/test_integrate_container_db_patch.py @@ -154,4 +154,4 @@ def test_dict_db_defective(self): } } ) - self.assertEqual(e.exception.message, "Not Supported on DictDB") + self.assertEqual(e.exception.message, "Not Supported iter function on DictDB") diff --git a/tests/test_container_db.py b/tests/test_container_db.py new file mode 100644 index 000000000..7bcb47d6b --- /dev/null +++ b/tests/test_container_db.py @@ -0,0 +1,284 @@ +import unittest + +import time +import plyvel + +from iconservice import VarDB, ArrayDB, DictDB, Address +from iconservice.database.db import KeyValueDatabase, ContextDatabase, IconScoreDatabase +from iconservice.icon_constant import IconScoreContextType +from iconservice.iconscore.icon_container_db import ContainerUtil +from iconservice.iconscore.icon_score_context import ContextContainer +from iconservice.iconscore.icon_score_context import IconScoreContext +from tests import create_address, rmtree +from tests.mock_db import MockKeyValueDatabase + +DB_PATH: str = ".mycom22_db" +VAR_DB: str = "test_var" +ARRAY_DB: str = "test_array" +DICT_DB1: str = "test_dict1" +DICT_DB2: str = "test_dict2" +SCORE_ADDR: 'Address' = create_address(1, b'0') + +REVISION: int = 10 +INDEX: int = 7 + +DISABLE = True + +# RANGE_LIST = [10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000, 5000000] +RANGE_LIST = [5000000] + +SCORE_ADDR_BYTES = SCORE_ADDR.to_bytes() + + +@unittest.skipIf(condition=DISABLE, reason="DISABLE") +class TestPlyvelDB(unittest.TestCase): + """ + Native PlyvelDB performance check + """ + + def _hash_key_bypass(self, key: bytes) -> bytes: + return key + + def _hash_key_origin(self, key: bytes) -> bytes: + data = [SCORE_ADDR.to_bytes()] + data.append(b'0x10') + data.append(key) + return b'|'.join(data) + + def _hash_key_cache_bytes(self, key: bytes) -> bytes: + data = [SCORE_ADDR_BYTES] + data.append(b'0x10') + data.append(key) + return b'|'.join(data) + + def _hash_key_cache_bytes_and_remove_append(self, key: bytes) -> bytes: + data = [SCORE_ADDR_BYTES, b'0x10', key] + return b'|'.join(data) + + def _put(self, range_cnt: int, hash_func: callable): + db = plyvel.DB(f"{DB_PATH}_{range_cnt}", create_if_missing=True) + + for i in range(range_cnt): + key = f"{i}".encode() + hashed_key = hash_func(key) + db.put(hashed_key, SCORE_ADDR_BYTES) + + def _get(self, range_cnt: int, hash_func: callable): + db = plyvel.DB(f"{DB_PATH}_{range_cnt}", create_if_missing=True) + + start = time.time() + + for i in range(range_cnt): + key = f"{i}".encode() + hashed_key = hash_func(key) + db.get(hashed_key) + + print(f"_get[{hash_func.__name__} {range_cnt} :", time.time() - start) + + def test_put(self): + for i in RANGE_LIST: + rmtree(f"{DB_PATH}_{i}") + + for i in RANGE_LIST: + self._put(i, self._hash_key_bypass) + + def test_get(self): + for i in RANGE_LIST: + self._get(i, self._hash_key_bypass) + + +@unittest.skipIf(condition=DISABLE, reason="DISABLE") +class TestPrebuildForContainerDB(unittest.TestCase): + """ + Prebuild DB for ContainerDB get + """ + + def _create_plyvel_db(self, range_cnt: int): + _db = KeyValueDatabase.from_path(f"{DB_PATH}{range_cnt}") + context_db = ContextDatabase(_db) + return IconScoreDatabase(SCORE_ADDR, context_db) + + def _create_new_db(self, range_cnt: int): + self.db = self._create_plyvel_db(range_cnt) + self._context = IconScoreContext(IconScoreContextType.DIRECT) + self._context.current_address = self.db.address + self._context.revision = REVISION + ContextContainer._push_context(self._context) + + ## LOGIC + + var_db = VarDB(VAR_DB, self.db, value_type=int) + array_db = ArrayDB(ARRAY_DB, self.db, value_type=Address) + dict_db1 = DictDB(DICT_DB1, self.db, value_type=Address) + dict_db2 = DictDB(DICT_DB2, self.db, value_type=int) + + index: int = 0 + for index in range(range_cnt): + addr: 'Address' = create_address() + array_db.put(addr) + dict_db1[index] = addr + dict_db2[addr] = index + var_db.set(index) + + ContextContainer._pop_context() + + def test_create_db(self): + for i in RANGE_LIST: + rmtree(f"{DB_PATH}{i}") + + for i in RANGE_LIST: + self._create_new_db(i) + + +def _create_plyvel_db(range_cnt: int): + _db = KeyValueDatabase.from_path(f"{DB_PATH}{range_cnt}") + context_db = ContextDatabase(_db) + return IconScoreDatabase(SCORE_ADDR, context_db) + + +def _create_mock_db(range_cnt: int): + mock_db = MockKeyValueDatabase.create_db() + context_db = ContextDatabase(mock_db) + return IconScoreDatabase(SCORE_ADDR, context_db) + + +# for profile +def _for_profile_function(range_cnt: int, _create_db_func: callable): + db = _create_db_func(range_cnt) + _context = IconScoreContext(IconScoreContextType.DIRECT) + _context.current_address = db.address + _context.revision = REVISION + ContextContainer._push_context(_context) + + array_db = ArrayDB(ARRAY_DB, db, value_type=Address) + + for index in range(range_cnt): + addr: 'Address' = create_address() + array_db.put(addr) + + for i in range(range_cnt): + a = array_db[i] + + ContextContainer._clear_context() + + +@unittest.skipIf(condition=DISABLE, reason="DISABLE") +class TestIconContainerDB(unittest.TestCase): + def _setup(self, range_cnt: int, _create_db_func: callable): + self.db = _create_db_func(range_cnt) + self._context = IconScoreContext(IconScoreContextType.DIRECT) + self._context.current_address = self.db.address + self._context.revision = REVISION + ContextContainer._push_context(self._context) + + def _tear_down(self): + ContextContainer._clear_context() + self.db = None + # rmtree(f"{DB_PATH}{range_cnt}") + + def _var_db_perfomance(self, + range_cnt: int, + _create_db_func: callable): + self._setup(range_cnt, _create_db_func) + + var_db = VarDB(VAR_DB, self.db, value_type=Address) + var_db.set(0) + + start = time.time() + + # LOGIC + for i in range(range_cnt): + a = var_db.get() + + print(f"_var_db_perfomance [{_create_db_func.__name__} {range_cnt} :", time.time() - start) + + self._tear_down() + + def _array_db_perfomance(self, + range_cnt: int, + _create_db_func: callable): + self._setup(range_cnt, _create_db_func) + + array_db = ArrayDB(ARRAY_DB, self.db, value_type=Address) + for index in range(range_cnt): + addr: 'Address' = create_address() + array_db.put(addr) + + start = time.time() + + # LOGIC + for i in range(range_cnt): + a = array_db[i] + + print(f"_array_db_perfomance [{_create_db_func.__name__} {range_cnt} :", time.time() - start) + + self._tear_down() + + def _dict_db_perfomance(self, + range_cnt: int, + _create_db_func: callable): + self._setup(range_cnt, _create_db_func) + + dict_db = DictDB(DICT_DB1, self.db, value_type=Address) + for index in range(range_cnt): + addr: 'Address' = create_address() + dict_db[index] = addr + start = time.time() + + # LOGIC + for i in range(range_cnt): + a = dict_db[i] + + print(f"_dict_db_perfomance [{_create_db_func.__name__} {range_cnt} :", time.time() - start) + + self._tear_down() + + def _complex_db_perfomance(self, + range_cnt: int, + _create_db_func: callable): + self._setup(range_cnt, _create_db_func) + + array_db = ArrayDB(ARRAY_DB, self.db, value_type=Address) + dict_db = DictDB(DICT_DB2, self.db, value_type=Address) + + for index in range(range_cnt): + addr: 'Address' = create_address() + array_db.put(addr) + dict_db[addr] = index + + start = time.time() + + # LOGIC + for i in range(range_cnt): + a = dict_db[array_db[0]] + + print(f"_complex_db_perfomance [{_create_db_func.__name__} {range_cnt} :", time.time() - start) + + self._tear_down() + + def test_var_db_performance(self): + for count in RANGE_LIST: + self._var_db_perfomance(count, _create_mock_db) + + def test_array_db_performance(self): + for count in RANGE_LIST: + self._array_db_perfomance(count, _create_mock_db) + + def test_dict_db_performance(self): + for count in RANGE_LIST: + self._dict_db_perfomance(count, _create_mock_db) + + def test_complex_db_performance(self): + for count in RANGE_LIST: + self._complex_db_perfomance(count, _create_mock_db) + + def test_profile(self): + from cProfile import Profile + from pstats import Stats + + # LOGIC + p = Profile() + p.runcall(_for_profile_function, 100_000, _create_mock_db) + + stats = Stats(p) + stats.print_stats() From 2a3f12a3cf16e92c2aaecbe92b06586f635e2e39 Mon Sep 17 00:00:00 2001 From: Chiwon Cho Date: Wed, 11 Mar 2020 18:41:14 +0900 Subject: [PATCH 2/2] Minor update in IconScoreDatabase * self.prefix_hash_key -> self._prefix_hash_key * Remove concatenation operator "+" from _make_prefix_hash_key() * Change an error message in DictDB: "Not Supported iter function on DictDB" -> "Iteration not supported in DictDB" --- iconservice/database/db.py | 18 +++++++++--------- iconservice/iconscore/icon_container_db.py | 2 +- .../test_integrate_container_db_patch.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/iconservice/database/db.py b/iconservice/database/db.py index 0ef427a4f..9b231cbdd 100644 --- a/iconservice/database/db.py +++ b/iconservice/database/db.py @@ -16,8 +16,8 @@ from typing import TYPE_CHECKING, Optional, Tuple, Iterable import plyvel - from iconcommons.logger import Logger + from .batch import TransactionBatchValue from ..base.exception import DatabaseException, InvalidParamsException, AccessDeniedException from ..icon_constant import ICON_DB_LOG_TAG @@ -384,13 +384,13 @@ def __init__(self, self._context_db = context_db self._observer: Optional[DatabaseObserver] = None - self.prefix_hash_key: bytes = self._make_prefix_hash_key() + self._prefix_hash_key: bytes = self._make_prefix_hash_key() def _make_prefix_hash_key(self) -> bytes: data = [self.address.to_bytes()] if self._prefix is not None: data.append(self._prefix) - return b'|'.join(data) + b'|' + return b'|'.join(data) def get(self, key: bytes) -> bytes: """ @@ -436,7 +436,7 @@ def get_sub_db(self, prefix: bytes) -> 'IconScoreSubDatabase': 'prefix is None in IconScoreDatabase.get_sub_db()') if self._prefix is not None: - prefix = b'|'.join([self._prefix, prefix]) + prefix = b'|'.join((self._prefix, prefix)) return IconScoreSubDatabase(self.address, self, prefix) @@ -469,7 +469,7 @@ def _hash_key(self, key: bytes) -> bytes: :return: key bytes """ - return b''.join((self.prefix_hash_key, key)) + return b'|'.join((self._prefix_hash_key, key)) def _validate_ownership(self): """Prevent a SCORE from accessing the database of another SCORE @@ -494,13 +494,13 @@ def __init__(self, address: 'Address', score_db: 'IconScoreDatabase', prefix: by self._prefix = prefix self._score_db = score_db - self.prefix_hash_key: bytes = self._make_prefix_hash_key() + self._prefix_hash_key: bytes = self._make_prefix_hash_key() def _make_prefix_hash_key(self) -> bytes: data = [] if self._prefix is not None: data.append(self._prefix) - return b'|'.join(data) + b'|' + return b'|'.join(data) def get(self, key: bytes) -> bytes: """ @@ -533,7 +533,7 @@ def get_sub_db(self, prefix: bytes) -> 'IconScoreSubDatabase': raise InvalidParamsException("Invalid prefix") if self._prefix is not None: - prefix = b'|'.join([self._prefix, prefix]) + prefix = b'|'.join((self._prefix, prefix)) return IconScoreSubDatabase(self.address, self._score_db, prefix) @@ -557,4 +557,4 @@ def _hash_key(self, key: bytes) -> bytes: :return: key bytes """ - return b''.join((self.prefix_hash_key, key)) + return b'|'.join((self._prefix_hash_key, key)) diff --git a/iconservice/iconscore/icon_container_db.py b/iconservice/iconscore/icon_container_db.py index 46310b261..1ecca476e 100644 --- a/iconservice/iconscore/icon_container_db.py +++ b/iconservice/iconscore/icon_container_db.py @@ -217,7 +217,7 @@ def __remove(self, key: K) -> None: self._db.delete(get_encoded_key(key)) def __iter__(self): - raise InvalidContainerAccessException("Not Supported iter function on DictDB") + raise InvalidContainerAccessException("Iteration not supported in DictDB") class ArrayDB(object): diff --git a/tests/integrate_test/test_integrate_container_db_patch.py b/tests/integrate_test/test_integrate_container_db_patch.py index d3b8a5c03..501405e0c 100644 --- a/tests/integrate_test/test_integrate_container_db_patch.py +++ b/tests/integrate_test/test_integrate_container_db_patch.py @@ -154,4 +154,4 @@ def test_dict_db_defective(self): } } ) - self.assertEqual(e.exception.message, "Not Supported iter function on DictDB") + self.assertEqual(e.exception.message, "Iteration not supported in DictDB")