fix all edsge case that could make pbdt chain stall

libraries/aleth/libp2p/ENR.cpp

@@ -204,7 +204,8 @@ PublicCompressed IdentitySchemeV4::publicKey(ENR const& _enr) {
 }
 
 std::ostream& operator<<(std::ostream& _out, ENR const& _enr) {
-  _out << "[ seq=" << _enr.sequenceNumber() << " " << "id=" << _enr.id() << " ";
+  _out << "[ seq=" << _enr.sequenceNumber() << " "
+       << "id=" << _enr.id() << " ";
 
   try {
     auto const pubKey = IdentitySchemeV4::publicKey(_enr);

libraries/aleth/libp2p/Host.cpp

@@ -137,7 +137,8 @@ Host::~Host() {
     }
   }
   // We need to poll both as strand_ is ioc_
-  while (0 < session_ioc_.poll() + ioc_.poll());
+  while (0 < session_ioc_.poll() + ioc_.poll())
+    ;
   save_state();
 
   ioc_.restart();

libraries/aleth/libp2p/NodeTable.h

@@ -416,8 +416,9 @@ struct NodeEntry {
 };
 
 inline std::ostream& operator<<(std::ostream& _out, NodeTable const& _nodeTable) {
-  _out << _nodeTable.m_hostNodeID << "\t" << "0\t" << _nodeTable.m_hostNodeEndpoint.address() << ":"
-       << _nodeTable.m_hostNodeEndpoint.udpPort() << std::endl;
+  _out << _nodeTable.m_hostNodeID << "\t"
+       << "0\t" << _nodeTable.m_hostNodeEndpoint.address() << ":" << _nodeTable.m_hostNodeEndpoint.udpPort()
+       << std::endl;
   auto s = _nodeTable.snapshot();
   for (auto n : s) _out << n.id() << "\t" << n.distance << "\t" << n.endpoint() << "\n";
   return _out;

libraries/cli/include/cli/config_jsons/default/default_genesis.json

@@ -118,8 +118,6 @@
     "ficus_hf": {
       "block_num": 50,
       "pillar_blocks_interval": 16,
-      "pillar_chain_sync_interval": 8,
-      "pbft_inclusion_delay": 6,
       "bridge_contract_address": "0xcAF2b453FE8382a4B8110356DF0508f6d71F22BF"
     }
   }

libraries/cli/include/cli/config_jsons/devnet/devnet_genesis.json

@@ -283,8 +283,6 @@
     "ficus_hf": {
       "block_num": 1000,
       "pillar_blocks_interval": 100,
-      "pillar_chain_sync_interval": 25,
-      "pbft_inclusion_delay": 6,
       "bridge_contract_address": "0xcAF2b453FE8382a4B8110356DF0508f6d71F22BF"
     }
   }

libraries/cli/include/cli/config_jsons/mainnet/mainnet_genesis.json

@@ -1652,8 +1652,6 @@
     "ficus_hf": {
       "block_num": -1,
       "pillar_blocks_interval": 100,
-      "pillar_chain_sync_interval": 25,
-      "pbft_inclusion_delay": 6,
       "bridge_contract_address": "0x0000000000000000000000000000000000000000"
     }
   }

libraries/common/include/common/encoding_rlp.hpp

@@ -67,8 +67,8 @@ void rlp(RLPEncoderRef encoding, std::pair<T1, T2> const& target) {
 }
 
 template <typename Sequence>
-auto rlp(RLPEncoderRef encoding, Sequence const& target) -> decltype(target.size(), target.begin(), target.end(),
-                                                                     void()) {
+auto rlp(RLPEncoderRef encoding, Sequence const& target)
+    -> decltype(target.size(), target.begin(), target.end(), void()) {
   encoding.appendList(target.size());
   for (auto const& v : target) {
     rlp(encoding, v);

libraries/config/include/config/hardfork.hpp

@@ -41,11 +41,7 @@ void dec_json(const Json::Value& json, AspenHardfork& obj);
 
 struct FicusHardforkConfig {
   uint64_t block_num{0};
-  uint64_t pillar_blocks_interval{100};     // [periods] how often is the new pillar block created
-  uint64_t pillar_chain_sync_interval{25};  // [periods] how often is pillar chain checked if it is in sync (has all
-                                            // previous pillar blocks and 2t+1 signatures for latest pillar block)
-  uint64_t pbft_inclusion_delay{
-      6};  // [periods] how many periods after the pillar block is created it is included in pbft block
+  uint64_t pillar_blocks_interval{100};    // [periods] how often is the new pillar block created
   taraxa::addr_t bridge_contract_address;  // [address] of the bridge contract
 
   bool isFicusHardfork(taraxa::PbftPeriod period) const;

libraries/config/src/hardfork.cpp

@@ -53,12 +53,14 @@ RLP_FIELDS_DEFINE(AspenHardfork, block_num_part_one, block_num_part_two, max_sup
 bool FicusHardforkConfig::isFicusHardfork(taraxa::PbftPeriod period) const { return period >= block_num; }
 
 bool FicusHardforkConfig::isPillarBlockPeriod(taraxa::PbftPeriod period, bool skip_first_pillar_block) const {
-  return period >= block_num && period >= firstPillarBlockPeriod() + (skip_first_pillar_block ? 1 : 0) * pillar_blocks_interval &&
+  return period >= block_num &&
+         period >= firstPillarBlockPeriod() + (skip_first_pillar_block ? 1 : 0) * pillar_blocks_interval &&
          period % pillar_blocks_interval == 0;
 }
 
 bool FicusHardforkConfig::isPbftWithPillarBlockPeriod(taraxa::PbftPeriod period) const {
-  return period >= firstPillarBlockPeriod() && period % pillar_blocks_interval == pbft_inclusion_delay;
+  // Pillar block hash is included in the next pbft block with period +1
+  return period >= firstPillarBlockPeriod() && period % pillar_blocks_interval == 1;
 }
 
 // Returns first pillar block period
@@ -72,43 +74,30 @@ void FicusHardforkConfig::validate(uint32_t delegation_delay) const {
     return;
   }
 
-  if (block_num % pillar_blocks_interval) {
-    throw taraxa::ConfigException("ficus_hf.block_num % ficus_hf.pillar_blocks_interval must == 0");
-  }
-
-  if (pillar_blocks_interval <= pillar_chain_sync_interval) {
-    throw taraxa::ConfigException("ficus_hf.pillar_blocks_interval must be > ficus_hf.pillar_chain_sync_interval");
+  if (pillar_blocks_interval <= 1) {
+    throw taraxa::ConfigException("ficus_hf.pillar_blocks_interval must be > 1");
   }
 
-  if (pillar_chain_sync_interval <= pbft_inclusion_delay) {
-    throw taraxa::ConfigException("ficus_hf.pillar_chain_sync_interval must be > ficus_hf.pbft_inclusion_delay");
-  }
-
-  if (pbft_inclusion_delay < 1 || pbft_inclusion_delay <= delegation_delay) {
-    throw taraxa::ConfigException("ficus_hf.pbft_inclusion_delay must be >= 1 && > dpos.delegation_delay");
+  if (block_num % pillar_blocks_interval) {
+    throw taraxa::ConfigException("ficus_hf.block_num % ficus_hf.pillar_blocks_interval must == 0");
   }
 }
 
 Json::Value enc_json(const FicusHardforkConfig& obj) {
   Json::Value json(Json::objectValue);
   json["block_num"] = dev::toJS(obj.block_num);
   json["pillar_blocks_interval"] = dev::toJS(obj.pillar_blocks_interval);
-  json["pillar_chain_sync_interval"] = dev::toJS(obj.pillar_chain_sync_interval);
-  json["pbft_inclusion_delay"] = dev::toJS(obj.pbft_inclusion_delay);
   json["bridge_contract_address"] = dev::toJS(obj.bridge_contract_address);
   return json;
 }
 
 void dec_json(const Json::Value& json, FicusHardforkConfig& obj) {
   obj.block_num = json["block_num"].isUInt64() ? dev::getUInt(json["block_num"]) : uint64_t(-1);
   obj.pillar_blocks_interval = dev::getUInt(json["pillar_blocks_interval"]);
-  obj.pillar_chain_sync_interval = dev::getUInt(json["pillar_chain_sync_interval"]);
-  obj.pbft_inclusion_delay = dev::getUInt(json["pbft_inclusion_delay"]);
   obj.bridge_contract_address = taraxa::addr_t(json["bridge_contract_address"].asString());
 }
 
-RLP_FIELDS_DEFINE(FicusHardforkConfig, block_num, pillar_blocks_interval, pillar_chain_sync_interval,
-                  pbft_inclusion_delay, bridge_contract_address)
+RLP_FIELDS_DEFINE(FicusHardforkConfig, block_num, pillar_blocks_interval, bridge_contract_address)
 
 // Json::Value enc_json(const BambooRedelegation& obj) {
 //   Json::Value json(Json::objectValue);

libraries/core_libs/consensus/include/dag/dag.hpp

@@ -130,7 +130,8 @@ class vertex_label_writer {
   vertex_label_writer(Property1 name) : name(name) {}
   template <class Vertex>
   void operator()(std::ostream &out, const Vertex &v) const {
-    out << "[label=\"" << name[v].toString().substr(0, 8) << " " << "\"]";
+    out << "[label=\"" << name[v].toString().substr(0, 8) << " "
+        << "\"]";
   }
 
  private:

libraries/core_libs/consensus/include/final_chain/final_chain.hpp

@@ -53,7 +53,7 @@ class FinalChain {
    * @param precommit_ext
    * @return finalization result
    */
-  virtual std::future<std::shared_ptr<FinalizationResult const>> finalize(
+  virtual std::future<std::shared_ptr<const FinalizationResult>> finalize(
       PeriodData&& period_data, std::vector<h256>&& finalized_dag_blk_hashes,
       std::shared_ptr<DagBlock>&& anchor = nullptr) = 0;
 
@@ -270,13 +270,10 @@ class FinalChain {
   }
 };
 
-std::shared_ptr<FinalChain> NewFinalChain(const std::shared_ptr<DB>& db, const taraxa::FullNodeConfig& config,
-                                          const addr_t& node_addr = {});
 /** @} */
 
 }  // namespace taraxa::final_chain
 
 namespace taraxa {
 using final_chain::FinalChain;
-using final_chain::NewFinalChain;
 }  // namespace taraxa

libraries/core_libs/consensus/include/final_chain/final_chain_impl.hpp

@@ -0,0 +1,109 @@
+#pragma once
+
+#include "final_chain/cache.hpp"
+#include "final_chain/final_chain.hpp"
+#include "rewards/rewards_stats.hpp"
+
+namespace taraxa::final_chain {
+
+class FinalChainImpl final : public FinalChain {
+  std::shared_ptr<DB> db_;
+  const uint64_t kBlockGasLimit;
+  StateAPI state_api_;
+  const bool kLightNode = false;
+  const uint32_t kMaxLevelsPerPeriod;
+  rewards::Stats rewards_;
+
+  // It is not prepared to use more then 1 thread. Examine it if you want to change threads count
+  boost::asio::thread_pool executor_thread_{1};
+
+  std::atomic<uint64_t> num_executed_dag_blk_ = 0;
+  std::atomic<uint64_t> num_executed_trx_ = 0;
+
+  EthBlockNumber delegation_delay_;
+
+  ValueByBlockCache<std::shared_ptr<const BlockHeader>> block_headers_cache_;
+  ValueByBlockCache<std::optional<const h256>> block_hashes_cache_;
+  ValueByBlockCache<const SharedTransactions> transactions_cache_;
+  ValueByBlockCache<std::shared_ptr<const TransactionHashes>> transaction_hashes_cache_;
+  MapByBlockCache<addr_t, std::optional<const state_api::Account>> accounts_cache_;
+
+  ValueByBlockCache<uint64_t> total_vote_count_cache_;
+  MapByBlockCache<addr_t, uint64_t> dpos_vote_count_cache_;
+  MapByBlockCache<addr_t, uint64_t> dpos_is_eligible_cache_;
+
+  std::condition_variable finalized_cv_;
+  std::mutex finalized_mtx_;
+
+  std::atomic<EthBlockNumber> last_block_number_;
+
+  const HardforksConfig& kHardforksConfig;
+  LOG_OBJECTS_DEFINE
+
+ public:
+  FinalChainImpl(const std::shared_ptr<DB>& db, const taraxa::FullNodeConfig& config, const addr_t& node_addr);
+
+  void stop() override;
+  std::future<std::shared_ptr<const FinalizationResult>> finalize(
+      PeriodData&& new_blk, std::vector<h256>&& finalized_dag_blk_hashes,
+      std::shared_ptr<DagBlock>&& anchor = nullptr) override;
+  EthBlockNumber delegation_delay() const override;
+  SharedTransaction make_bridge_finalization_transaction();
+  bool isNeedToFinalize(EthBlockNumber blk_num) const;
+  std::vector<SharedTransaction> makeSystemTransactions(PbftPeriod blk_num);
+  std::shared_ptr<const FinalizationResult> finalize_(PeriodData&& new_blk,
+                                                      std::vector<h256>&& finalized_dag_blk_hashes,
+                                                      std::shared_ptr<DagBlock>&& anchor);
+  void prune(EthBlockNumber blk_n) override;
+  std::shared_ptr<BlockHeader> append_block(DB::Batch& batch, const addr_t& author, uint64_t timestamp,
+                                            uint64_t gas_limit, const h256& state_root, u256 total_reward,
+                                            const SharedTransactions& transactions = {},
+                                            const TransactionReceipts& receipts = {}, const bytes& extra_data = {});
+  EthBlockNumber last_block_number() const override;
+  std::optional<EthBlockNumber> block_number(const h256& h) const override;
+  std::optional<h256> block_hash(std::optional<EthBlockNumber> n = {}) const override;
+  std::shared_ptr<const BlockHeader> block_header(std::optional<EthBlockNumber> n = {}) const override;
+  std::optional<TransactionLocation> transaction_location(const h256& trx_hash) const override;
+  std::optional<TransactionReceipt> transaction_receipt(const h256& trx_h) const override;
+  uint64_t transactionCount(std::optional<EthBlockNumber> n = {}) const override;
+  std::shared_ptr<const TransactionHashes> transaction_hashes(std::optional<EthBlockNumber> n = {}) const override;
+  const SharedTransactions transactions(std::optional<EthBlockNumber> n = {}) const override;
+  std::vector<EthBlockNumber> withBlockBloom(const LogBloom& b, EthBlockNumber from, EthBlockNumber to) const override;
+  std::optional<state_api::Account> get_account(const addr_t& addr,
+                                                std::optional<EthBlockNumber> blk_n = {}) const override;
+  void update_state_config(const state_api::Config& new_config) override;
+  h256 get_account_storage(const addr_t& addr, const u256& key,
+                           std::optional<EthBlockNumber> blk_n = {}) const override;
+  bytes get_code(const addr_t& addr, std::optional<EthBlockNumber> blk_n = {}) const override;
+  state_api::ExecutionResult call(const state_api::EVMTransaction& trx,
+                                  std::optional<EthBlockNumber> blk_n = {}) const override;
+  std::string trace(std::vector<state_api::EVMTransaction> trxs, EthBlockNumber blk_n,
+                    std::optional<state_api::Tracing> params = {}) const override;
+  uint64_t dpos_eligible_total_vote_count(EthBlockNumber blk_num) const override;
+  uint64_t dpos_eligible_vote_count(EthBlockNumber blk_num, const addr_t& addr) const override;
+  bool dpos_is_eligible(EthBlockNumber blk_num, const addr_t& addr) const override;
+  vrf_wrapper::vrf_pk_t dpos_get_vrf_key(EthBlockNumber blk_n, const addr_t& addr) const override;
+  std::vector<state_api::ValidatorStake> dpos_validators_total_stakes(EthBlockNumber blk_num) const override;
+  virtual uint256_t dpos_total_amount_delegated(EthBlockNumber blk_num) const override;
+  std::vector<state_api::ValidatorVoteCount> dpos_validators_vote_counts(EthBlockNumber blk_num) const override;
+  void wait_for_finalized() override;
+  uint64_t dpos_yield(EthBlockNumber blk_num) const override;
+  u256 dpos_total_supply(EthBlockNumber blk_num) const override;
+  h256 get_bridge_root(EthBlockNumber blk_num) const override;
+  h256 get_bridge_epoch(EthBlockNumber blk_num) const override;
+
+ private:
+  std::shared_ptr<TransactionHashes> get_transaction_hashes(std::optional<EthBlockNumber> n = {}) const;
+  const SharedTransactions get_transactions(std::optional<EthBlockNumber> n = {}) const;
+  std::shared_ptr<const BlockHeader> get_block_header(EthBlockNumber n) const;
+  std::optional<h256> get_block_hash(EthBlockNumber n) const;
+  EthBlockNumber last_if_absent(const std::optional<EthBlockNumber>& client_blk_n) const;
+  static state_api::EVMTransaction to_evm_transaction(const SharedTransaction& trx);
+  static void append_evm_transactions(std::vector<state_api::EVMTransaction>& evm_trxs, const SharedTransactions& trxs);
+  BlocksBlooms block_blooms(const h256& chunk_id) const;
+  static h256 block_blooms_chunk_id(EthBlockNumber level, EthBlockNumber index);
+  std::vector<EthBlockNumber> withBlockBloom(const LogBloom& b, EthBlockNumber from, EthBlockNumber to,
+                                             EthBlockNumber level, EthBlockNumber index) const;
+};
+
+}  // namespace taraxa::final_chain

libraries/core_libs/consensus/include/pbft/pbft_manager.hpp

@@ -598,6 +598,7 @@ class PbftManager {
   bool already_next_voted_null_block_hash_ = false;
   bool go_finish_state_ = false;
   bool loop_back_finish_state_ = false;
+  bool already_placed_pillar_vote_ = false;
 
   // Used to avoid cyclic logging in voting steps that are called repeatedly
   bool printSecondFinishStepInfo_ = true;

libraries/core_libs/consensus/include/pillar_chain/pillar_chain_manager.hpp

@@ -44,31 +44,23 @@ class PillarChainManager {
   /**
    * @Process Creates new pillar block
    *
-   * @param block_data
+   * @param block_header
    * @return pillar block in case block was created, otherwise nullptr
    */
-  std::shared_ptr<PillarBlock> createPillarBlock(const std::shared_ptr<final_chain::FinalizationResult>& block_data);
+  std::shared_ptr<PillarBlock> createPillarBlock(const std::shared_ptr<const final_chain::BlockHeader>& block_header);
 
   /**
    * @brief Generate and place pillar vote for provided pillar_block_hash in case the whole pillar block is present and
    * valid
    *
+   * @param period
    * @param pillar_block_hash
    * @param node_sk
-   * @param is_pbft_syncing
+   * @param broadcast_vote
    * @return true if vote placed, otherwise false
    */
-  bool genAndPlacePillarVote(const blk_hash_t& pillar_block_hash, const secret_t& node_sk, bool is_pbft_syncing);
-
-  /**
-   * @brief Check if pillar chain is synced - node has all previous pillar blocks(+votes) and there is > threshold
-   * votes for latest pillar block. If not, request them
-   *
-   * @param block_num - current block number
-   *
-   * @return true if synced, otherwise false
-   */
-  bool checkPillarChainSynced(EthBlockNumber block_num) const;
+  bool genAndPlacePillarVote(PbftPeriod period, const blk_hash_t& pillar_block_hash, const secret_t& node_sk,
+                             bool broadcast_vote);
 
   /**
    * @brief Set network as a weak pointer

libraries/core_libs/consensus/include/pillar_chain/pillar_votes.hpp

@@ -39,14 +39,6 @@ class PillarVotes {
    */
   bool isUniqueVote(const std::shared_ptr<PillarVote> vote) const;
 
-  /**
-   * @brief Checks if there above threshold votes for specified period
-   *
-   * @param period
-   * @return
-   */
-  bool hasAboveThresholdVotes(PbftPeriod period, const blk_hash_t& block_hash) const;
-
   /**
    * @brief Checks if specified period data have been already initialized
    *