diff --git a/iconservice/database/db.py b/iconservice/database/db.py index 3a430eed8..0ef427a4f 100644 --- a/iconservice/database/db.py +++ b/iconservice/database/db.py @@ -384,6 +384,14 @@ def __init__(self, self._context_db = context_db self._observer: Optional[DatabaseObserver] = None + self.prefix_hash_key: bytes = self._make_prefix_hash_key() + + def _make_prefix_hash_key(self) -> bytes: + data = [self.address.to_bytes()] + if self._prefix is not None: + data.append(self._prefix) + return b'|'.join(data) + b'|' + def get(self, key: bytes) -> bytes: """ Gets the value for the specified key @@ -460,12 +468,8 @@ def _hash_key(self, key: bytes) -> bytes: :params key: key passed by SCORE :return: key bytes """ - data = [self.address.to_bytes()] - if self._prefix is not None: - data.append(self._prefix) - data.append(key) - return b'|'.join(data) + return b''.join((self.prefix_hash_key, key)) def _validate_ownership(self): """Prevent a SCORE from accessing the database of another SCORE @@ -490,6 +494,14 @@ def __init__(self, address: 'Address', score_db: 'IconScoreDatabase', prefix: by self._prefix = prefix self._score_db = score_db + self.prefix_hash_key: bytes = self._make_prefix_hash_key() + + def _make_prefix_hash_key(self) -> bytes: + data = [] + if self._prefix is not None: + data.append(self._prefix) + return b'|'.join(data) + b'|' + def get(self, key: bytes) -> bytes: """ Gets the value for the specified key @@ -544,9 +556,5 @@ def _hash_key(self, key: bytes) -> bytes: :params key: key passed by SCORE :return: key bytes """ - data = [] - if self._prefix is not None: - data.append(self._prefix) - data.append(key) - return b'|'.join(data) + return b''.join((self.prefix_hash_key, key)) diff --git a/iconservice/iconscore/icon_container_db.py b/iconservice/iconscore/icon_container_db.py index 4b2be31c1..01f37bd41 100644 --- a/iconservice/iconscore/icon_container_db.py +++ b/iconservice/iconscore/icon_container_db.py @@ -16,10 +16,8 @@ from typing import TypeVar, Optional, Any, Union, TYPE_CHECKING -from .icon_score_context import ContextContainer from ..base.address import Address from ..base.exception import InvalidParamsException, InvalidContainerAccessException -from ..icon_constant import Revision, IconScoreContextType from ..utils import int_to_bytes, bytes_to_int if TYPE_CHECKING: @@ -39,27 +37,27 @@ def get_encoded_key(key: V) -> bytes: class ContainerUtil(object): - @staticmethod - def create_db_prefix(cls, var_key: K) -> bytes: + @classmethod + def create_db_prefix(cls, container_cls: type, var_key: K) -> bytes: """Create a prefix used as a parameter of IconScoreDatabase.get_sub_db() - :param cls: ArrayDB, DictDB, VarDB + :param container_cls: ArrayDB, DictDB, VarDB :param var_key: :return: """ - if cls == ArrayDB: + if container_cls == ArrayDB: container_id = ARRAY_DB_ID - elif cls == DictDB: + elif container_cls == DictDB: container_id = DICT_DB_ID else: - raise InvalidParamsException(f'Unsupported container class: {cls}') + raise InvalidParamsException(f'Unsupported container class: {container_cls}') encoded_key: bytes = get_encoded_key(var_key) return b'|'.join([container_id, encoded_key]) - @staticmethod - def encode_key(key: K) -> bytes: + @classmethod + def encode_key(cls, key: K) -> bytes: """Create a key passed to IconScoreDatabase :param key: @@ -80,8 +78,8 @@ def encode_key(key: K) -> bytes: raise InvalidParamsException(f'Unsupported key type: {type(key)}') return bytes_key - @staticmethod - def encode_value(value: V) -> bytes: + @classmethod + def encode_value(cls, value: V) -> bytes: if isinstance(value, int): byte_value = int_to_bytes(value) elif isinstance(value, str): @@ -96,8 +94,8 @@ def encode_value(value: V) -> bytes: raise InvalidParamsException(f'Unsupported value type: {type(value)}') return byte_value - @staticmethod - def decode_object(value: bytes, value_type: type) -> Optional[Union[K, V]]: + @classmethod + def decode_object(cls, value: bytes, value_type: type) -> Optional[Union[K, V]]: if value is None: return get_default_value(value_type) @@ -114,45 +112,45 @@ def decode_object(value: bytes, value_type: type) -> Optional[Union[K, V]]: obj_value = value return obj_value - @staticmethod - def remove_prefix_from_iters(iter_items: iter) -> iter: - return ((ContainerUtil.__remove_prefix_from_key(key), value) for key, value in iter_items) + @classmethod + def remove_prefix_from_iters(cls, iter_items: iter) -> iter: + return ((cls.__remove_prefix_from_key(key), value) for key, value in iter_items) - @staticmethod - def __remove_prefix_from_key(key_from_bytes: bytes) -> bytes: + @classmethod + def __remove_prefix_from_key(cls, key_from_bytes: bytes) -> bytes: return key_from_bytes[:-1] - @staticmethod - def put_to_db(db: 'IconScoreDatabase', db_key: str, container: iter) -> None: - sub_db = db.get_sub_db(ContainerUtil.encode_key(db_key)) + @classmethod + def put_to_db(cls, db: 'IconScoreDatabase', db_key: str, container: iter) -> None: + sub_db = db.get_sub_db(cls.encode_key(db_key)) if isinstance(container, dict): - ContainerUtil.__put_to_db_internal(sub_db, container.items()) + cls.__put_to_db_internal(sub_db, container.items()) elif isinstance(container, (list, set, tuple)): - ContainerUtil.__put_to_db_internal(sub_db, enumerate(container)) + cls.__put_to_db_internal(sub_db, enumerate(container)) - @staticmethod - def get_from_db(db: 'IconScoreDatabase', db_key: str, *args, value_type: type) -> Optional[K]: - sub_db = db.get_sub_db(ContainerUtil.encode_key(db_key)) + @classmethod + def get_from_db(cls, db: 'IconScoreDatabase', db_key: str, *args, value_type: type) -> Optional[K]: + sub_db = db.get_sub_db(cls.encode_key(db_key)) *args, last_arg = args for arg in args: - sub_db = sub_db.get_sub_db(ContainerUtil.encode_key(arg)) + sub_db = sub_db.get_sub_db(cls.encode_key(arg)) - byte_key = sub_db.get(ContainerUtil.encode_key(last_arg)) + byte_key = sub_db.get(cls.encode_key(last_arg)) if byte_key is None: return get_default_value(value_type) - return ContainerUtil.decode_object(byte_key, value_type) + return cls.decode_object(byte_key, value_type) - @staticmethod - def __put_to_db_internal(db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], iters: iter) -> None: + @classmethod + def __put_to_db_internal(cls, db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], iters: iter) -> None: for key, value in iters: - sub_db = db.get_sub_db(ContainerUtil.encode_key(key)) + sub_db = db.get_sub_db(cls.encode_key(key)) if isinstance(value, dict): - ContainerUtil.__put_to_db_internal(sub_db, value.items()) + cls.__put_to_db_internal(sub_db, value.items()) elif isinstance(value, (list, set, tuple)): - ContainerUtil.__put_to_db_internal(sub_db, enumerate(value)) + cls.__put_to_db_internal(sub_db, enumerate(value)) else: - db_key = ContainerUtil.encode_key(key) - db_value = ContainerUtil.encode_value(value) + db_key = cls.encode_key(key) + db_value = cls.encode_value(value) db.put(db_key, db_value) @@ -232,7 +230,7 @@ def __init__(self, var_key: K, db: 'IconScoreDatabase', value_type: type) -> Non prefix: bytes = ContainerUtil.create_db_prefix(type(self), var_key) self._db = db.get_sub_db(prefix) self.__value_type = value_type - self.__legacy_size = self.__get_size_from_db() + self.__size = self.__get_size_from_db() def put(self, value: V) -> None: """ @@ -240,7 +238,7 @@ def put(self, value: V) -> None: :param value: value to add """ - size: int = self.__get_size() + size: int = self.__size self.__put(size, value) self.__set_size(size + 1) @@ -250,7 +248,7 @@ def pop(self) -> Optional[V]: :return: last added value """ - size: int = self.__get_size() + size: int = self.__size if size == 0: return None @@ -269,35 +267,29 @@ def get(self, index: int = 0) -> V: """ return self[index] - def __get_size(self) -> int: - if self.__is_defective_revision(): - return self.__legacy_size - else: - return self.__get_size_from_db() - def __get_size_from_db(self) -> int: - return ContainerUtil.decode_object(self._db.get(ArrayDB.__SIZE_BYTE_KEY), int) + return ContainerUtil.decode_object(self._db.get(self.__SIZE_BYTE_KEY), int) def __set_size(self, size: int) -> None: - self.__legacy_size = size + self.__size = size byte_value = ContainerUtil.encode_value(size) - self._db.put(ArrayDB.__SIZE_BYTE_KEY, byte_value) + self._db.put(self.__SIZE_BYTE_KEY, byte_value) def __put(self, index: int, value: V) -> None: byte_value = ContainerUtil.encode_value(value) self._db.put(get_encoded_key(index), byte_value) def __iter__(self): - return self._get_generator(self._db, self.__get_size(), self.__value_type) + return self._get_generator(self._db, self.__size, self.__value_type) def __len__(self): - return self.__get_size() + return self.__size def __setitem__(self, index: int, value: V) -> None: if not isinstance(index, int): raise InvalidParamsException('Invalid index type: not an integer') - size: int = self.__get_size() + size: int = self.__size # Negative index means that you count from the right instead of the left. if index < 0: @@ -309,7 +301,7 @@ def __setitem__(self, index: int, value: V) -> None: raise InvalidParamsException('ArrayDB out of index') def __getitem__(self, index: int) -> V: - return ArrayDB._get(self._db, self.__get_size(), index, self.__value_type) + return self._get(self._db, self.__size, index, self.__value_type) def __contains__(self, item: V): for e in self: @@ -317,14 +309,8 @@ def __contains__(self, item: V): return True return False - @staticmethod - def __is_defective_revision(): - context = ContextContainer._get_context() - revision = context.revision - return context.type == IconScoreContextType.INVOKE and revision < Revision.THREE.value - - @staticmethod - def _get(db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, index: int, value_type: type) -> V: + @classmethod + def _get(cls, db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, index: int, value_type: type) -> V: if not isinstance(index, int): raise InvalidParamsException('Invalid index type: not an integer') @@ -338,10 +324,10 @@ def _get(db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, inde raise InvalidParamsException('ArrayDB out of index') - @staticmethod - def _get_generator(db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, value_type: type): + @classmethod + def _get_generator(cls, db: Union['IconScoreDatabase', 'IconScoreSubDatabase'], size: int, value_type: type): for index in range(size): - yield ArrayDB._get(db, size, index, value_type) + yield cls._get(db, size, index, value_type) class VarDB(object): diff --git a/iconservice/iconscore/icon_score_mapper_object.py b/iconservice/iconscore/icon_score_mapper_object.py index 44ee5079e..526925f15 100644 --- a/iconservice/iconscore/icon_score_mapper_object.py +++ b/iconservice/iconscore/icon_score_mapper_object.py @@ -67,7 +67,7 @@ def get_score(self, revision: int) -> 'IconScoreBase': :param revision: :return: """ - if revision <= Revision.TWO.value or self.address == GOVERNANCE_SCORE_ADDRESS: + if revision <= Revision.TWO.value: if self._score is None: self._score = self.create_score() diff --git a/tests/test_container_db.py b/tests/test_container_db.py new file mode 100644 index 000000000..84640b387 --- /dev/null +++ b/tests/test_container_db.py @@ -0,0 +1,284 @@ +import unittest + +import time +import plyvel + +from iconservice import VarDB, ArrayDB, DictDB, Address +from iconservice.database.db import KeyValueDatabase, ContextDatabase, IconScoreDatabase +from iconservice.icon_constant import IconScoreContextType +from iconservice.iconscore.icon_container_db import ContainerUtil +from iconservice.iconscore.icon_score_context import ContextContainer +from iconservice.iconscore.icon_score_context import IconScoreContext +from tests import create_address, rmtree +from tests.mock_db import MockKeyValueDatabase + +DB_PATH: str = ".mycom22_db" +VAR_DB: str = "test_var" +ARRAY_DB: str = "test_array" +DICT_DB1: str = "test_dict1" +DICT_DB2: str = "test_dict2" +SCORE_ADDR: 'Address' = create_address(1, b'0') + +REVISION: int = 10 +INDEX: int = 7 + +DISABLE = False + +# RANGE_LIST = [10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000, 5000000] +RANGE_LIST = [5000000] + +SCORE_ADDR_BYTES = SCORE_ADDR.to_bytes() + + +@unittest.skipIf(condition=DISABLE, reason="DISABLE") +class TestPlyvelDB(unittest.TestCase): + """ + Native PlyvelDB performance check + """ + + def _hash_key_bypass(self, key: bytes) -> bytes: + return key + + def _hash_key_origin(self, key: bytes) -> bytes: + data = [SCORE_ADDR.to_bytes()] + data.append(b'0x10') + data.append(key) + return b'|'.join(data) + + def _hash_key_cache_bytes(self, key: bytes) -> bytes: + data = [SCORE_ADDR_BYTES] + data.append(b'0x10') + data.append(key) + return b'|'.join(data) + + def _hash_key_cache_bytes_and_remove_append(self, key: bytes) -> bytes: + data = [SCORE_ADDR_BYTES, b'0x10', key] + return b'|'.join(data) + + def _put(self, range_cnt: int, hash_func: callable): + db = plyvel.DB(f"{DB_PATH}_{range_cnt}", create_if_missing=True) + + for i in range(range_cnt): + key = f"{i}".encode() + hashed_key = hash_func(key) + db.put(hashed_key, SCORE_ADDR_BYTES) + + def _get(self, range_cnt: int, hash_func: callable): + db = plyvel.DB(f"{DB_PATH}_{range_cnt}", create_if_missing=True) + + start = time.time() + + for i in range(range_cnt): + key = f"{i}".encode() + hashed_key = hash_func(key) + db.get(hashed_key) + + print(f"_get[{hash_func.__name__} {range_cnt} :", time.time() - start) + + def test_put(self): + for i in RANGE_LIST: + rmtree(f"{DB_PATH}_{i}") + + for i in RANGE_LIST: + self._put(i, self._hash_key_bypass) + + def test_get(self): + for i in RANGE_LIST: + self._get(i, self._hash_key_bypass) + + +@unittest.skipIf(condition=DISABLE, reason="DISABLE") +class TestPrebuildForContainerDB(unittest.TestCase): + """ + Prebuild DB for ContainerDB get + """ + + def _create_plyvel_db(self, range_cnt: int): + _db = KeyValueDatabase.from_path(f"{DB_PATH}{range_cnt}") + context_db = ContextDatabase(_db) + return IconScoreDatabase(SCORE_ADDR, context_db) + + def _create_new_db(self, range_cnt: int): + self.db = self._create_plyvel_db(range_cnt) + self._context = IconScoreContext(IconScoreContextType.DIRECT) + self._context.current_address = self.db.address + self._context.revision = REVISION + ContextContainer._push_context(self._context) + + ## LOGIC + + var_db = VarDB(VAR_DB, self.db, value_type=int) + array_db = ArrayDB(ARRAY_DB, self.db, value_type=Address) + dict_db1 = DictDB(DICT_DB1, self.db, value_type=Address) + dict_db2 = DictDB(DICT_DB2, self.db, value_type=int) + + index: int = 0 + for index in range(range_cnt): + addr: 'Address' = create_address() + array_db.put(addr) + dict_db1[index] = addr + dict_db2[addr] = index + var_db.set(index) + + ContextContainer._pop_context() + + def test_create_db(self): + for i in RANGE_LIST: + rmtree(f"{DB_PATH}{i}") + + for i in RANGE_LIST: + self._create_new_db(i) + + +def _create_plyvel_db(range_cnt: int): + _db = KeyValueDatabase.from_path(f"{DB_PATH}{range_cnt}") + context_db = ContextDatabase(_db) + return IconScoreDatabase(SCORE_ADDR, context_db) + + +def _create_mock_db(range_cnt: int): + mock_db = MockKeyValueDatabase.create_db() + context_db = ContextDatabase(mock_db) + return IconScoreDatabase(SCORE_ADDR, context_db) + + +# for profile +def _for_profile_function(range_cnt: int, _create_db_func: callable): + db = _create_db_func(range_cnt) + _context = IconScoreContext(IconScoreContextType.DIRECT) + _context.current_address = db.address + _context.revision = REVISION + ContextContainer._push_context(_context) + + array_db = ArrayDB(ARRAY_DB, db, value_type=Address) + + for index in range(range_cnt): + addr: 'Address' = create_address() + array_db.put(addr) + + for i in range(range_cnt): + a = array_db[i] + + ContextContainer._clear_context() + + +@unittest.skipIf(condition=DISABLE, reason="DISABLE") +class TestIconContainerDB(unittest.TestCase): + def _setup(self, range_cnt: int, _create_db_func: callable): + self.db = _create_db_func(range_cnt) + self._context = IconScoreContext(IconScoreContextType.DIRECT) + self._context.current_address = self.db.address + self._context.revision = REVISION + ContextContainer._push_context(self._context) + + def _tear_down(self): + ContextContainer._clear_context() + self.db = None + # rmtree(f"{DB_PATH}{range_cnt}") + + def _var_db_perfomance(self, + range_cnt: int, + _create_db_func: callable): + self._setup(range_cnt, _create_db_func) + + var_db = VarDB(VAR_DB, self.db, value_type=Address) + var_db.set(0) + + start = time.time() + + # LOGIC + for i in range(range_cnt): + a = var_db.get() + + print(f"_var_db_perfomance [{_create_db_func.__name__} {range_cnt} :", time.time() - start) + + self._tear_down() + + def _array_db_perfomance(self, + range_cnt: int, + _create_db_func: callable): + self._setup(range_cnt, _create_db_func) + + array_db = ArrayDB(ARRAY_DB, self.db, value_type=Address) + for index in range(range_cnt): + addr: 'Address' = create_address() + array_db.put(addr) + + start = time.time() + + # LOGIC + for i in range(range_cnt): + a = array_db[i] + + print(f"_array_db_perfomance [{_create_db_func.__name__} {range_cnt} :", time.time() - start) + + self._tear_down() + + def _dict_db_perfomance(self, + range_cnt: int, + _create_db_func: callable): + self._setup(range_cnt, _create_db_func) + + dict_db = DictDB(DICT_DB1, self.db, value_type=Address) + for index in range(range_cnt): + addr: 'Address' = create_address() + dict_db[index] = addr + start = time.time() + + # LOGIC + for i in range(range_cnt): + a = dict_db[i] + + print(f"_dict_db_perfomance [{_create_db_func.__name__} {range_cnt} :", time.time() - start) + + self._tear_down() + + def _complex_db_perfomance(self, + range_cnt: int, + _create_db_func: callable): + self._setup(range_cnt, _create_db_func) + + array_db = ArrayDB(ARRAY_DB, self.db, value_type=Address) + dict_db = DictDB(DICT_DB2, self.db, value_type=Address) + + for index in range(range_cnt): + addr: 'Address' = create_address() + array_db.put(addr) + dict_db[addr] = index + + start = time.time() + + # LOGIC + for i in range(range_cnt): + a = dict_db[array_db[0]] + + print(f"_complex_db_perfomance [{_create_db_func.__name__} {range_cnt} :", time.time() - start) + + self._tear_down() + + def test_var_db_performance(self): + for count in RANGE_LIST: + self._var_db_perfomance(count, _create_mock_db) + + def test_array_db_performance(self): + for count in RANGE_LIST: + self._array_db_perfomance(count, _create_mock_db) + + def test_dict_db_performance(self): + for count in RANGE_LIST: + self._dict_db_perfomance(count, _create_mock_db) + + def test_complex_db_performance(self): + for count in RANGE_LIST: + self._complex_db_perfomance(count, _create_mock_db) + + def test_profile(self): + from cProfile import Profile + from pstats import Stats + + # LOGIC + p = Profile() + p.runcall(_for_profile_function, 100_000, _create_mock_db) + + stats = Stats(p) + stats.print_stats()