Merge pull request #397 from icon-project/release-1.5.20

Release 1.5.20

VERSION

@@ -1 +1 @@
-1.5.18
+1.5.20

iconservice/icon_constant.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from enum import IntFlag, unique, IntEnum, Enum, auto
+from enum import IntFlag, unique, IntEnum, Enum, auto, Flag
 
 GOVERNANCE_ADDRESS = "cx0000000000000000000000000000000000000001"
 
@@ -104,20 +104,18 @@ class IssueDataKey:
 
 
 class Revision(Enum):
-    def _generate_next_value_(self, start, count, last_values):
-        if self != 'LATEST':
-            return start + count + 1
+    TWO = 2
+    THREE = 3
+    FOUR = 4
+    IISS = 5
+    DECENTRALIZATION = 6
+    FIX_TOTAL_ELECTED_PREP_DELEGATED = 7
 
-        return last_values[-1]
+    # Revision 8
+    REALTIME_P2P_ENDPOINT_UPDATE = 8
+    OPTIMIZE_DIRTY_PREP_UPDATE = 8
 
-    TWO = auto()
-    THREE = auto()
-    FOUR = auto()
-    IISS = auto()
-    DECENTRALIZATION = auto()
-    IS_1_5_16 = auto()
-
-    LATEST = auto()
+    LATEST = 8
 
 
 RC_DB_VERSION_0 = 0
@@ -165,7 +163,7 @@ class ConfigKey:
     STEP_TRACE_FLAG = 'stepTraceFlag'
     PRECOMMIT_DATA_LOG_FLAG = 'precommitDataLogFlag'
 
-    #icon rc
+    # Reward calculator executable path
     ICON_RC_DIR_PATH = 'iconRcPath'
 
     # IISS meta data
@@ -344,3 +342,45 @@ class BlockVoteStatus(Enum):
     NONE = 0
     TRUE = 1
     FALSE = 2
+
+
+class PRepFlag(Flag):
+    """Setting flags to True means that PRep fields specified by the flags has been changed
+
+    """
+    NONE = 0
+    STATUS = auto()
+    NAME = auto()
+    COUNTRY = auto()
+    CITY = auto()
+    EMAIL = auto()
+    WEBSITE = auto()
+    DETAILS = auto()
+    P2P_ENDPOINT = auto()
+    PENALTY = auto()
+    GRADE = auto()
+    STAKE = auto()
+    DELEGATED = auto()
+    LAST_GENERATE_BLOCK_HEIGHT = auto()
+    TOTAL_BLOCKS = auto()
+    VALIDATED_BLOCKS = auto()
+    UNVALIDATED_SEQUENCE_BLOCKS = auto()
+    IREP = auto()  # irep, irep_block_height
+    IREP_BLOCK_HEIGHT = auto()
+
+    BLOCK_STATISTICS = TOTAL_BLOCKS | VALIDATED_BLOCKS | UNVALIDATED_SEQUENCE_BLOCKS
+    ALL = 0xFFFFFFFF
+
+
+class PRepContainerFlag(Flag):
+    NONE = 0
+    DIRTY = auto()
+
+
+class TermFlag(Flag):
+    NONE = 0
+    MAIN_PREPS = auto()
+    SUB_PREPS = auto()
+    MAIN_PREP_P2P_ENDPOINT = auto()
+
+    ALL = 0xFFFFFFFF

iconservice/icon_inner_service.py

@@ -12,21 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from asyncio import get_event_loop
+import asyncio
 from concurrent.futures.thread import ThreadPoolExecutor
 from typing import Any, TYPE_CHECKING, Optional, Tuple
 
 from earlgrey import message_queue_task, MessageQueueStub, MessageQueueService
-
 from iconcommons.logger import Logger
+
 from iconservice.base.address import Address
 from iconservice.base.block import Block
 from iconservice.base.exception import ExceptionCode, IconServiceBaseException, InvalidBaseTransactionException, \
     FatalException, ServiceNotReadyException
 from iconservice.base.type_converter import TypeConverter, ParamType
 from iconservice.base.type_converter_templates import ConstantKeys
 from iconservice.icon_constant import ICON_INNER_LOG_TAG, ICON_SERVICE_LOG_TAG, \
-    EnableThreadFlag, ENABLE_THREAD_FLAG, ConfigKey, RCStatus
+    EnableThreadFlag, ENABLE_THREAD_FLAG
 from iconservice.icon_service_engine import IconServiceEngine
 from iconservice.utils import check_error_response, to_camel_case
 
@@ -39,6 +39,9 @@
 THREAD_VALIDATE = 'validate'
 
 
+_TAG = "IIS"
+
+
 class IconScoreInnerTask(object):
     def __init__(self, conf: 'IconConfig'):
         self._conf = conf
@@ -75,7 +78,7 @@ async def hello(self):
         await ready_future
 
         if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
-            loop = get_event_loop()
+            loop = asyncio.get_event_loop()
             ret = await loop.run_in_executor(self._thread_pool[THREAD_INVOKE], self._hello)
         else:
             ret = self._hello()
@@ -88,25 +91,31 @@ def _hello(self):
         return self._icon_service_engine.hello()
 
     def _close(self):
-        Logger.info("icon_score_service close", ICON_INNER_LOG_TAG)
+        Logger.info(tag=_TAG, msg="_close() start")
 
         if self._icon_service_engine:
             self._icon_service_engine.close()
             self._icon_service_engine = None
         MessageQueueService.loop.stop()
 
+        Logger.info(tag=_TAG, msg="_close() end")
+
     @message_queue_task
     async def close(self):
+        Logger.info(tag=_TAG, msg="close() start")
+
         self._close()
 
+        Logger.info(tag=_TAG, msg="close() end")
+
     @message_queue_task
     async def invoke(self, request: dict):
         Logger.info(f'invoke request with {request}', ICON_INNER_LOG_TAG)
 
         self._check_icon_service_ready()
 
         if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
-            loop = get_event_loop()
+            loop = asyncio.get_event_loop()
             return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
                                               self._invoke, request)
         else:
@@ -184,7 +193,7 @@ async def query(self, request: dict):
         self._check_icon_service_ready()
 
         if self._is_thread_flag_on(EnableThreadFlag.QUERY):
-            loop = get_event_loop()
+            loop = asyncio.get_event_loop()
             return await loop.run_in_executor(self._thread_pool[THREAD_QUERY],
                                               self._query, request)
         else:
@@ -227,7 +236,7 @@ async def call(self, request: dict):
         self._check_icon_service_ready()
 
         if self._is_thread_flag_on(EnableThreadFlag.QUERY):
-            loop = get_event_loop()
+            loop = asyncio.get_event_loop()
             return await loop.run_in_executor(self._thread_pool[THREAD_QUERY],
                                               self._call, request)
         else:
@@ -261,7 +270,7 @@ async def write_precommit_state(self, request: dict):
         self._check_icon_service_ready()
 
         if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
-            loop = get_event_loop()
+            loop = asyncio.get_event_loop()
             return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
                                               self._write_precommit_state, request)
         else:
@@ -309,7 +318,7 @@ async def remove_precommit_state(self, request: dict):
         self._check_icon_service_ready()
 
         if self._is_thread_flag_on(EnableThreadFlag.INVOKE):
-            loop = get_event_loop()
+            loop = asyncio.get_event_loop()
             return await loop.run_in_executor(self._thread_pool[THREAD_INVOKE],
                                               self._remove_precommit_state, request)
         else:
@@ -345,7 +354,7 @@ async def validate_transaction(self, request: dict):
         self._check_icon_service_ready()
 
         if self._is_thread_flag_on(EnableThreadFlag.VALIDATE):
-            loop = get_event_loop()
+            loop = asyncio.get_event_loop()
             return await loop.run_in_executor(self._thread_pool[THREAD_VALIDATE],
                                               self._validate_transaction, request)
         else:

iconservice/icon_service_engine.py

@@ -2129,18 +2129,26 @@ def hello(self) -> dict:
         """
         Logger.debug(tag=self.TAG, msg="hello() start")
 
+        iiss_engine: 'IISSEngine' = IconScoreContext.engine.iiss
+
+        last_block: 'Block' = self._get_last_block()
+
         if isinstance(self._wal_reader, WriteAheadLogReader):
             wal_state = WALState(self._wal_reader.state)
 
             # If only writing rc_db is done on commit without sending COMMIT_BLOCK to rc,
             # send COMMIT_BLOCK to rc prior to invoking a block
             if not (wal_state & WALState.SEND_COMMIT_BLOCK):
-                IconScoreContext.engine.iiss.send_commit(
+                iiss_engine.send_commit(
                     self._wal_reader.block.height, self._wal_reader.instant_block_hash)
 
+            assert last_block == self._wal_reader.block
+
             # No need to use
             self._wal_reader = None
 
+        # iiss_engine.init_reward_calculator(last_block)
+
         Logger.debug(tag=self.TAG, msg="hello() end")
 
         return {}

iconservice/iconscore/icon_score_base2.py

@@ -329,7 +329,6 @@ def get_main_prep_info() -> Tuple[List[PRepInfo], int]:
     context = ContextContainer._get_context()
     assert context
 
-    # TODO: Fix an error on unittest first before removing the commit below (goldworm)
     term = context.term
     if term is None:
         return [], -1

iconservice/iconscore/icon_score_context.py

@@ -20,7 +20,7 @@
 from typing import TYPE_CHECKING, Optional, List
 
 from iconcommons.logger import Logger
-from iconservice.icx.issue.regulator import Regulator
+
 from .icon_score_mapper import IconScoreMapper
 from .icon_score_trace import Trace
 from ..base.block import Block
@@ -29,8 +29,9 @@
 from ..base.transaction import Transaction
 from ..database.batch import BlockBatch, TransactionBatch
 from ..icon_constant import (
-    IconScoreContextType, IconScoreFuncType, TERM_PERIOD, PRepGrade, PREP_MAIN_PREPS, PREP_MAIN_AND_SUB_PREPS
-)
+    IconScoreContextType, IconScoreFuncType, TERM_PERIOD, PRepGrade, PREP_MAIN_PREPS, PREP_MAIN_AND_SUB_PREPS,
+    Revision, PRepFlag)
+from ..icx.issue.regulator import Regulator
 
 if TYPE_CHECKING:
     from .icon_score_base import IconScoreBase
@@ -185,13 +186,6 @@ def preps(self) -> Optional['PRepContainer']:
     def term(self) -> Optional['Term']:
         return self._term
 
-    def is_term_updated(self) -> bool:
-        """Returns whether info in self._term is changed
-
-        :return:
-        """
-        return self._term and self._term.is_dirty()
-
     def is_decentralized(self) -> bool:
         return self.engine.prep.term is not None
 
@@ -231,7 +225,7 @@ def update_rc_db_batch(self):
         self.rc_tx_batch.clear()
 
     def update_dirty_prep_batch(self):
-        """Update context.preps when a tx is done
+        """Apply updated P-Rep data to context.preps every time when a tx is done
 
         Caution: call update_dirty_prep_batch before update_state_db_batch()
         """
@@ -240,6 +234,9 @@ def update_dirty_prep_batch(self):
             # we should update P-Reps in this term
             self._update_elected_preps_in_term(dirty_prep)
 
+            if self.revision >= Revision.REALTIME_P2P_ENDPOINT_UPDATE.value:
+                self._update_main_preps_in_term(dirty_prep)
+
             self._preps.replace(dirty_prep)
             # Write serialized dirty_prep data into tx_batch
             self.storage.prep.put_prep(self, dirty_prep)
@@ -266,10 +263,24 @@ def _update_elected_preps_in_term(self, dirty_prep: 'PRep'):
         # Just in case, reset the P-Rep grade one to CANDIDATE
         dirty_prep.grade = PRepGrade.CANDIDATE
 
-        self._term.update_preps(self.revision, [dirty_prep])
+        self._term.update_invalid_elected_preps([dirty_prep])
 
         Logger.info(tag=self.TAG, msg=f"Invalid main and sub prep: {dirty_prep}")
 
+    def _update_main_preps_in_term(self, dirty_prep: 'PRep'):
+        """
+
+        :param dirty_prep: dirty prep
+        """
+        if self._term is None:
+            return
+
+        if dirty_prep.is_flags_on(PRepFlag.P2P_ENDPOINT) and \
+                self._term.is_main_prep(dirty_prep.address):
+            self._term.on_main_prep_p2p_endpoint_updated()
+
+        Logger.info(tag=self.TAG, msg=f"_update_main_prep_endpoint_in_term: {dirty_prep}")
+
     def clear_batch(self):
         if self.tx_batch:
             self.tx_batch.clear()
@@ -295,10 +306,19 @@ def get_prep(self, address: 'Address', mutable: bool = False) -> Optional['PRep'
     def put_dirty_prep(self, prep: 'PRep'):
         Logger.debug(tag=self.TAG, msg=f"put_dirty_prep() start: {prep}")
 
-        if self._tx_dirty_preps is not None:
-            self._tx_dirty_preps[prep.address] = prep
+        if self._tx_dirty_preps is None:
+            Logger.warning(tag=self.TAG, msg="self._tx_dirty_preps is None")
+            Logger.debug(tag=self.TAG, msg="put_dirty_prep() end")
+            return
+
+        if not prep.is_dirty() and self.revision >= Revision.OPTIMIZE_DIRTY_PREP_UPDATE.value:
+            Logger.info(tag=self.TAG, msg=f"No need to update an unchanged P-Rep: revision={self.revision}")
+            Logger.debug(tag=self.TAG, msg="put_dirty_prep() end")
+            return
+
+        self._tx_dirty_preps[prep.address] = prep
 
-        Logger.debug(tag=self.TAG, msg=f"put_dirty_prep() end")
+        Logger.debug(tag=self.TAG, msg="put_dirty_prep() end")
 
 
 class IconScoreContextFactory(object):

iconservice/iconscore/icon_score_engine.py

@@ -111,7 +111,10 @@ def _call(context: 'IconScoreContext',
         context.current_address: 'Address' = icon_score_address
 
         score_func = getattr(icon_score, ATTR_SCORE_CALL)
-        return score_func(func_name=func_name, kw_params=converted_params)
+        ret = score_func(func_name=func_name, kw_params=converted_params)
+
+        # No problem even though ret is None
+        return deepcopy(ret)
 
     @staticmethod
     def _convert_score_params_by_annotations(icon_score: 'IconScoreBase', func_name: str, kw_params: dict) -> dict: