sync pillar votes after restart

libraries/config/include/config/hardfork.hpp

@@ -45,6 +45,14 @@ struct FicusHardforkConfig {
   uint64_t pillar_chain_sync_periods{25};  // [periods] how often is pillar chain checked if it is in sync (has all
                                            // previous pillar blocks and 2t+1 signatures for latest pillar block)
 
+  bool isPillarBlockPeriod(taraxa::PbftPeriod period, uint64_t from_n_th_period = 2) const {
+    return period >= from_n_th_period * pillar_block_periods && period % pillar_block_periods == 0;
+  }
+
+  bool isPillarBlockPeriodPlusN(taraxa::PbftPeriod period, taraxa::PbftPeriod n) const {
+    return period >= pillar_block_periods && period % pillar_block_periods == n;
+  }
+
   HAS_RLP_FIELDS
 };
 Json::Value enc_json(const FicusHardforkConfig& obj);

libraries/core_libs/consensus/src/pbft/pbft_manager.cpp

@@ -581,6 +581,12 @@ void PbftManager::broadcastVotes() {
 
       LOG(log_dg_) << "Broadcast own votes for period " << period << ", round " << round;
     }
+
+    // Broadcast own pillar vote
+    if (const auto &own_pillar_vote = db_->getOwnPillarBlockVote(); own_pillar_vote) {
+      LOG(log_er_) << "remove me: Broadcast own pillar vote " << own_pillar_vote->getHash() << " for period " << own_pillar_vote->getPeriod();
+      net->gossipPillarBlockVote(own_pillar_vote, rebroadcast);
+    }
   };
 
   const auto round_elapsed_time = elapsedTimeInMs(current_round_start_datetime_);
@@ -599,12 +605,20 @@ void PbftManager::broadcastVotes() {
   } else if (period_elapsed_time / kMinLambda > kRebroadcastVotesLambdaTime * rebroadcast_reward_votes_counter_) {
     // Stalled in the same period for kRebroadcastVotesLambdaTime * kMinLambda time -> rebroadcast reward votes
     gossipVotes(vote_mgr_->getRewardVotes(), "2t+1 propose reward votes", true);
+    // Broadcast own pillar vote
+    if (const auto &own_pillar_vote = db_->getOwnPillarBlockVote(); own_pillar_vote) {
+      net->gossipPillarBlockVote(own_pillar_vote, true);
+    }
     rebroadcast_reward_votes_counter_++;
     // If there was a rebroadcast no need to do next broadcast either
     broadcast_reward_votes_counter_++;
   } else if (period_elapsed_time / kMinLambda > kBroadcastVotesLambdaTime * broadcast_reward_votes_counter_) {
     // Stalled in the same period for kBroadcastVotesLambdaTime * kMinLambda time -> broadcast reward votes
     gossipVotes(vote_mgr_->getRewardVotes(), "2t+1 propose reward votes", false);
+    // Broadcast own pillar vote
+    if (const auto &own_pillar_vote = db_->getOwnPillarBlockVote(); own_pillar_vote) {
+      net->gossipPillarBlockVote(own_pillar_vote, false);
+    }
     broadcast_reward_votes_counter_++;
   }
 }
@@ -918,20 +932,26 @@ void PbftManager::certifyBlock_() {
     return;
   }
 
+  // Generate pillar vote in case pillar block hash is present in pbft block. Do not broadcast cert vote in case pillar
+  // vote was not created & broadcast
+  if (const auto pillar_block_hash = soft_voted_block->getPillarBlockHash(); pillar_block_hash.has_value()) {
+    // TODO: validate and save pillar_block_hash in pbft_manager, then try to vote for it during each period if node did
+    // not vote yet, otherwise it might not vote ever, if this block was pushed through syncing
+
+    // Creates pillar vote
+    if (!pillar_chain_mgr_->genAndPlacePillarVote(*pillar_block_hash, node_sk_)) {
+      LOG(log_er_) << "Failed to create pillar vote for " << *pillar_block_hash;
+      return;
+    }
+  }
+
   if (!placeVote(vote, "cert vote", soft_voted_block)) {
     LOG(log_er_) << "Failed to place cert vote for " << soft_voted_block->getBlockHash();
     return;
   }
 
   cert_voted_block_for_round_ = soft_voted_block;
   db_->saveCertVotedBlockInRound(round, soft_voted_block);
-
-  // Generate pillar vote in case pillar block hash is present in pbft block
-  if (const auto pillar_block_hash = soft_voted_block->getPillarBlockHash(); pillar_block_hash.has_value()) {
-    // TODO: validate and save pillar_block_hash in pbft_manager, then try to vote for it during each period if node did
-    // not vote yet, otherwise it might not vote ever, if this block was pushed through syncing Creates pillar vote
-    pillar_chain_mgr_->genAndPlacePillarVote(*pillar_block_hash, node_sk_);
-  }
 }
 
 void PbftManager::firstFinish_() {
@@ -1164,9 +1184,8 @@ PbftManager::proposePbftBlock() {
   }
 
   std::optional<blk_hash_t> pillar_block_hash;
-  if (current_pbft_period >= kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods &&
-      current_pbft_period % kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods ==
-          kGenesisConfig.state.dpos.delegation_delay) {
+  if (kGenesisConfig.state.hardforks.ficus_hf.isPillarBlockPeriodPlusN(current_pbft_period,
+                                                                       kGenesisConfig.state.dpos.delegation_delay)) {
     // Anchor pillar block into the pbft block
     const auto pillar_block = pillar_chain_mgr_->getCurrentPillarBlock();
     if (!pillar_block) {
@@ -1382,8 +1401,7 @@ bool PbftManager::validatePbftBlock(const std::shared_ptr<PbftBlock> &pbft_block
   const auto kBlockPeriod = pbft_block->getPeriod();
 
   // Check if pillar chain is synced
-  if (kBlockPeriod >= 2 * kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods &&
-      kBlockPeriod % kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods == 0) {
+  if (kGenesisConfig.state.hardforks.ficus_hf.isPillarBlockPeriod(kBlockPeriod)) {
     if (!pillar_chain_mgr_->checkPillarChainSynced(kBlockPeriod)) {
       LOG(log_er_) << "Unable to validate pbft block " << pbft_block_hash << ", period " << kBlockPeriod
                    << ". Pillar chain is not synced";
@@ -1393,9 +1411,8 @@ bool PbftManager::validatePbftBlock(const std::shared_ptr<PbftBlock> &pbft_block
 
   // Validate optional pillar block hash
   if (const auto pillar_block_hash = pbft_block->getPillarBlockHash();
-      kBlockPeriod >= kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods &&
-      kBlockPeriod % kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods ==
-          kGenesisConfig.state.dpos.delegation_delay) {
+      kGenesisConfig.state.hardforks.ficus_hf.isPillarBlockPeriodPlusN(kBlockPeriod,
+                                                                       kGenesisConfig.state.dpos.delegation_delay)) {
     if (!pillar_block_hash.has_value()) {
       LOG(log_er_) << "PBFT block " << pbft_block_hash << " does not contain pillar block hash, period "
                    << kBlockPeriod;
@@ -1804,9 +1821,8 @@ std::optional<std::pair<PeriodData, std::vector<std::shared_ptr<PbftVote>>>> Pbf
 
   // Validate pillar block hash
   const auto kBlockPeriod = period_data.pbft_blk->getPeriod();
-  if (kBlockPeriod > kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods &&
-      kBlockPeriod % kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods ==
-          kGenesisConfig.state.dpos.delegation_delay) {
+  if (kGenesisConfig.state.hardforks.ficus_hf.isPillarBlockPeriodPlusN(kBlockPeriod,
+                                                                       kGenesisConfig.state.dpos.delegation_delay)) {
     if (!period_data.pbft_blk->getPillarBlockHash().has_value()) {
       LOG(log_er_) << "Synced PBFT block " << pbft_block_hash << ", period " << kBlockPeriod
                    << " does not contain pillar block hash";
@@ -1825,8 +1841,7 @@ std::optional<std::pair<PeriodData, std::vector<std::shared_ptr<PbftVote>>>> Pbf
   }
 
   // Validate pillar votes
-  if (kBlockPeriod >= 2 * kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods &&
-      kBlockPeriod % kGenesisConfig.state.hardforks.ficus_hf.pillar_block_periods == 0) {
+  if (kGenesisConfig.state.hardforks.ficus_hf.isPillarBlockPeriod(kBlockPeriod)) {
     if (!period_data.pillar_votes_.has_value()) {
       LOG(log_er_) << "Synced PBFT block " << pbft_block_hash << ", period " << kBlockPeriod
                    << " does not contain pillar votes";

libraries/core_libs/consensus/src/pillar_chain/pillar_chain_manager.cpp

@@ -43,7 +43,6 @@ PillarChainManager::PillarChainManager(const FicusHardforkConfig& ficusHfConfig,
     last_finalized_pillar_block_ = std::move(latest_pillar_block_data->block_);
   }
 
-  // TODO: implement mchanism similar to pbft votes when the pillar votes are sent in case pbft is stuck
   LOG_OBJECTS_CREATE("PILLAR_CHAIN");
 }
 
@@ -193,8 +192,6 @@ std::shared_ptr<PillarBlock> PillarChainManager::getCurrentPillarBlock() const {
 bool PillarChainManager::checkPillarChainSynced(EthBlockNumber block_num) const {
   std::shared_lock<std::shared_mutex> lock(mutex_);
 
-  LOG(log_er_) << "checkPillarChainSynced called for period " << block_num;
-
   // No current pillar block registered... This should happen only before the first pillar block is created
   if (!current_pillar_block_) [[unlikely]] {
     if (block_num > kFicusHfConfig.pillar_block_periods) {
@@ -220,7 +217,7 @@ bool PillarChainManager::checkPillarChainSynced(EthBlockNumber block_num) const
   }
 
   PbftPeriod expected_pillar_block_period = 0;
-  if (block_num % kFicusHfConfig.pillar_block_periods == 0) {
+  if (kFicusHfConfig.isPillarBlockPeriod(block_num, 1)) {
     expected_pillar_block_period = block_num - kFicusHfConfig.pillar_block_periods;
   } else {
     expected_pillar_block_period = block_num - (block_num % kFicusHfConfig.pillar_block_periods);

libraries/core_libs/network/include/network/network.hpp

@@ -65,7 +65,7 @@ class Network {
   void gossipVote(const std::shared_ptr<PbftVote> &vote, const std::shared_ptr<PbftBlock> &block,
                   bool rebroadcast = false);
   void gossipVotesBundle(const std::vector<std::shared_ptr<PbftVote>> &votes, bool rebroadcast = false);
-  void gossipPillarBlockVote(const std::shared_ptr<PillarVote> &vote);
+  void gossipPillarBlockVote(const std::shared_ptr<PillarVote> &vote, bool rebroadcast = false);
   void handleMaliciousSyncPeer(const dev::p2p::NodeID &id);
   std::shared_ptr<network::tarcap::TaraxaPeer> getMaxChainPeer() const;
 

libraries/core_libs/network/src/network.cpp

@@ -299,9 +299,9 @@ void Network::gossipVotesBundle(const std::vector<std::shared_ptr<PbftVote>> &vo
   }
 }
 
-void Network::gossipPillarBlockVote(const std::shared_ptr<PillarVote> &vote) {
+void Network::gossipPillarBlockVote(const std::shared_ptr<PillarVote> &vote, bool rebroadcast) {
   for (const auto &tarcap : tarcaps_) {
-    tarcap.second->getSpecificHandler<network::tarcap::PillarVotePacketHandler>()->onNewPillarVote(vote);
+    tarcap.second->getSpecificHandler<network::tarcap::PillarVotePacketHandler>()->onNewPillarVote(vote, rebroadcast);
   }
 }
 

libraries/core_libs/network/src/tarcap/packets_handlers/latest/get_pbft_sync_packet_handler.cpp

@@ -88,8 +88,7 @@ void GetPbftSyncPacketHandler::sendPbftBlocks(const std::shared_ptr<TaraxaPeer>
     // TODO: bad solution: should not decode PeriodData, add pillar votes and then encode it...
     PeriodData period_data{data};
     // Add pillar votes to period data
-    if (block_period >= 2 * kConf.genesis.state.hardforks.ficus_hf.pillar_block_periods &&
-        block_period % kConf.genesis.state.hardforks.ficus_hf.pillar_block_periods == 0) {
+    if (kConf.genesis.state.hardforks.ficus_hf.isPillarBlockPeriod(block_period)) {
       auto pillar_data =
           db_->getPillarBlockData(block_period - kConf.genesis.state.hardforks.ficus_hf.pillar_block_periods);
       if (!pillar_data.has_value()) {

libraries/core_libs/network/src/tarcap/packets_handlers/latest/pbft_sync_packet_handler.cpp

@@ -134,8 +134,7 @@ void PbftSyncPacketHandler::process(const threadpool::PacketData &packet_data,
 
     // Validate optional pillar block hash
     const auto kFicusHfConfig = kConf.genesis.state.hardforks.ficus_hf;
-    if (pbft_block_period > kFicusHfConfig.pillar_block_periods &&
-        pbft_block_period % kFicusHfConfig.pillar_block_periods == kConf.genesis.state.dpos.delegation_delay) {
+    if (kFicusHfConfig.isPillarBlockPeriodPlusN(pbft_block_period, kConf.genesis.state.dpos.delegation_delay)) {
       if (!period_data.pbft_blk->getPillarBlockHash().has_value()) {
         LOG(log_er_) << "Synced PBFT block " << pbft_blk_hash << ", period " << pbft_block_period
                      << " does not contain pillar block hash";
@@ -152,8 +151,7 @@ void PbftSyncPacketHandler::process(const threadpool::PacketData &packet_data,
     }
 
     // Validate optional pillar votes
-    if (pbft_block_period >= 2 * kFicusHfConfig.pillar_block_periods &&
-        pbft_block_period % kFicusHfConfig.pillar_block_periods == 0) {
+    if (kFicusHfConfig.isPillarBlockPeriod(pbft_block_period)) {
       if (!period_data.pillar_votes_.has_value()) {
         LOG(log_er_) << "Synced PBFT block " << pbft_blk_hash << ", period " << pbft_block_period
                      << " does not contain pillar votes";