chore: optimize migration

libraries/core_libs/consensus/include/final_chain/data.hpp

@@ -24,7 +24,6 @@ using LogBlooms = std::vector<LogBloom>;
 using Nonce = dev::h64;
 
 struct BlockHeaderData {
-  h256 hash;
   h256 parent_hash;
   h256 state_root;
   h256 transactions_root;
@@ -42,6 +41,7 @@ struct BlockHeader : BlockHeaderData {
   BlockHeader() = default;
   BlockHeader(std::string&& raw_header_data);
   BlockHeader(std::string&& raw_header_data, const PbftBlock& pbft, uint64_t gas_limit);
+  h256 hash;
   Address author;
   uint64_t gas_limit = 0;
   uint64_t timestamp = 0;
@@ -58,7 +58,7 @@ struct BlockHeader : BlockHeaderData {
 
   static h256 const& mixHash();
 
-  dev::bytes&& ethereumRlp() const;
+  dev::bytes ethereumRlp() const;
 };
 
 static constexpr auto c_bloomIndexSize = 16;

libraries/core_libs/consensus/include/final_chain/final_chain.hpp

@@ -18,8 +18,6 @@ namespace taraxa::final_chain {
  * @{
  */
 
-enum class DBMetaKeys { LAST_NUMBER = 1 };
-
 /**
  * @brief main responsibility is blocks execution in EVM, getting data from EVM state
  *
@@ -284,6 +282,8 @@ class FinalChain {
   std::vector<SharedTransaction> makeSystemTransactions(PbftPeriod blk_num);
 
   std::shared_ptr<BlockHeader> makeGenesisHeader(std::string&& raw_header) const;
+  std::shared_ptr<BlockHeader> makeGenesisHeader(const h256& state_root) const;
+
   std::shared_ptr<BlockHeader> appendBlock(Batch& batch, const PbftBlock& pbft_blk, const h256& state_root,
                                            u256 total_reward, const SharedTransactions& transactions = {},
                                            const TransactionReceipts& receipts = {});
@@ -322,7 +322,7 @@ class FinalChain {
 
   std::atomic<EthBlockNumber> last_block_number_;
 
-  const HardforksConfig& kHardforksConfig;
+  const FullNodeConfig& kConfig;
   LOG_OBJECTS_DEFINE
 };
 

libraries/core_libs/consensus/src/final_chain/data.cpp

@@ -10,7 +10,7 @@ namespace taraxa::final_chain {
 
 dev::bytes BlockHeaderData::serializeForDB() const { return util::rlp_enc(*this); }
 
-RLP_FIELDS_DEFINE(BlockHeaderData, hash, parent_hash, state_root, transactions_root, receipts_root, log_bloom, gas_used,
+RLP_FIELDS_DEFINE(BlockHeaderData, parent_hash, state_root, transactions_root, receipts_root, log_bloom, gas_used,
                   total_reward)
 
 BlockHeader::BlockHeader(std::string&& raw_header_data)
@@ -20,6 +20,7 @@ BlockHeader::BlockHeader(std::string&& raw_header_data_, const PbftBlock& pbft_,
     : BlockHeader(std::move(raw_header_data_)) {
   setFromPbft(pbft_);
   gas_limit = gas_limit_;
+  hash = dev::sha3(ethereumRlp());
 }
 
 void BlockHeader::setFromPbft(const PbftBlock& pbft) {
@@ -37,7 +38,7 @@ u256 const& BlockHeader::difficulty() { return ZeroU256(); }
 
 h256 const& BlockHeader::mixHash() { return ZeroHash(); }
 
-dev::bytes&& BlockHeader::ethereumRlp() const {
+dev::bytes BlockHeader::ethereumRlp() const {
   dev::RLPStream rlp_strm;
   util::rlp_tuple(rlp_strm, parent_hash, BlockHeader::unclesHash(), author, state_root, transactions_root,
                   receipts_root, log_bloom, BlockHeader::difficulty(), number, gas_limit, gas_used, timestamp,

libraries/core_libs/consensus/src/final_chain/final_chain.cpp

@@ -37,7 +37,7 @@ FinalChain::FinalChain(const std::shared_ptr<DbStorage>& db, const taraxa::FullN
       dpos_is_eligible_cache_(
           config.final_chain_cache_in_blocks,
           [this](uint64_t blk, const addr_t& addr) { return state_api_.dpos_is_eligible(blk, addr); }),
-      kHardforksConfig(config.genesis.state.hardforks) {
+      kConfig(config) {
   LOG_OBJECTS_CREATE("EXECUTOR");
   num_executed_dag_blk_ = db_->getStatusField(taraxa::StatusDbField::ExecutedBlkCount);
   num_executed_trx_ = db_->getStatusField(taraxa::StatusDbField::ExecutedTrxCount);
@@ -46,10 +46,8 @@ FinalChain::FinalChain(const std::shared_ptr<DbStorage>& db, const taraxa::FullN
   // If we don't have genesis block in db then create and push it
   if (!last_blk_num) [[unlikely]] {
     auto batch = db_->createWriteBatch();
-    auto header = std::make_shared<BlockHeader>();
-    header->timestamp = config.genesis.dag_genesis_block.getTimestamp();
-    header->state_root = state_db_descriptor.state_root;
-    appendBlock(batch, header);
+    auto header = makeGenesisHeader(state_db_descriptor.state_root);
+    appendBlock(batch, header, {}, {});
 
     block_headers_cache_.append(header->number, header);
     last_block_number_ = header->number;
@@ -118,13 +116,14 @@ SharedTransaction FinalChain::makeBridgeFinalizationTransaction() {
   auto account = getAccount(kTaraxaSystemAccount).value_or(state_api::ZeroAccount);
 
   auto trx = std::make_shared<SystemTransaction>(account.nonce, 0, 0, kBlockGasLimit, finalize_method,
-                                                 kHardforksConfig.ficus_hf.bridge_contract_address);
+                                                 kConfig.genesis.state.hardforks.ficus_hf.bridge_contract_address);
   return trx;
 }
 
 bool FinalChain::isNeedToFinalize(EthBlockNumber blk_num) const {
   const static auto get_bridge_root_method = util::EncodingSolidity::packFunctionCall("shouldFinalizeEpoch()");
-  return u256(call(state_api::EVMTransaction{dev::ZeroAddress, 1, kHardforksConfig.ficus_hf.bridge_contract_address,
+  return u256(call(state_api::EVMTransaction{dev::ZeroAddress, 1,
+                                             kConfig.genesis.state.hardforks.ficus_hf.bridge_contract_address,
                                              state_api::ZeroAccount.nonce, 0, 10000000, get_bridge_root_method},
                    blk_num)
                   .code_retval)
@@ -136,8 +135,9 @@ std::vector<SharedTransaction> FinalChain::makeSystemTransactions(PbftPeriod blk
   // Make system transactions <delegationDelay()> blocks sooner than next pillar block period,
   // e.g.: if pillar block period is 100, this will return true for period 100 - delegationDelay() == 95, 195, 295,
   // etc...
-  if (kHardforksConfig.ficus_hf.isPillarBlockPeriod(blk_num + delegationDelay())) {
-    if (const auto bridge_contract = getAccount(kHardforksConfig.ficus_hf.bridge_contract_address); bridge_contract) {
+  if (kConfig.genesis.state.hardforks.ficus_hf.isPillarBlockPeriod(blk_num + delegationDelay())) {
+    if (const auto bridge_contract = getAccount(kConfig.genesis.state.hardforks.ficus_hf.bridge_contract_address);
+        bridge_contract) {
       if (bridge_contract->code_size && isNeedToFinalize(blk_num - 1)) {
         auto finalize_trx = makeBridgeFinalizationTransaction();
         system_transactions.push_back(finalize_trx);
@@ -314,7 +314,7 @@ std::shared_ptr<BlockHeader> FinalChain::appendBlock(Batch& batch, const PbftBlo
   header->total_reward = total_reward;
   header->gas_limit = kBlockGasLimit;
 
-  return appendBlock(batch, header, transactions, receipts);
+  return appendBlock(batch, std::move(header), transactions, receipts);
 }
 
 std::shared_ptr<BlockHeader> FinalChain::appendBlock(Batch& batch, std::shared_ptr<BlockHeader> header,
@@ -337,7 +337,6 @@ std::shared_ptr<BlockHeader> FinalChain::appendBlock(Batch& batch, std::shared_p
 
   header->transactions_root = hash256(trxs_trie);
   header->receipts_root = hash256(receipts_trie);
-
   header->hash = dev::sha3(header->ethereumRlp());
 
   auto data = header->serializeForDB();
@@ -504,15 +503,17 @@ u256 FinalChain::dposTotalSupply(EthBlockNumber blk_num) const { return state_ap
 
 h256 FinalChain::getBridgeRoot(EthBlockNumber blk_num) const {
   const static auto get_bridge_root_method = util::EncodingSolidity::packFunctionCall("getBridgeRoot()");
-  return h256(call(state_api::EVMTransaction{dev::ZeroAddress, 1, kHardforksConfig.ficus_hf.bridge_contract_address,
+  return h256(call(state_api::EVMTransaction{dev::ZeroAddress, 1,
+                                             kConfig.genesis.state.hardforks.ficus_hf.bridge_contract_address,
                                              state_api::ZeroAccount.nonce, 0, 10000000, get_bridge_root_method},
                    blk_num)
                   .code_retval);
 }
 
 h256 FinalChain::getBridgeEpoch(EthBlockNumber blk_num) const {
   const static auto getBridgeEpoch_method = util::EncodingSolidity::packFunctionCall("finalizedEpoch()");
-  return h256(call(state_api::EVMTransaction{dev::ZeroAddress, 1, kHardforksConfig.ficus_hf.bridge_contract_address,
+  return h256(call(state_api::EVMTransaction{dev::ZeroAddress, 1,
+                                             kConfig.genesis.state.hardforks.ficus_hf.bridge_contract_address,
                                              state_api::ZeroAccount.nonce, 0, 10000000, getBridgeEpoch_method},
                    blk_num)
                   .code_retval);
@@ -546,12 +547,21 @@ const SharedTransactions FinalChain::getTransactions(std::optional<EthBlockNumbe
 
 std::shared_ptr<BlockHeader> FinalChain::makeGenesisHeader(std::string&& raw_header) const {
   auto bh = std::make_shared<BlockHeader>(std::move(raw_header));
-  bh->gas_limit = kBlockGasLimit;
-  // bh->timestamp = config.genesis.dag_genesis_block.getTimestamp();
-  bh->number = 0;
+  bh->gas_limit = kConfig.genesis.pbft.gas_limit;
+  bh->timestamp = kConfig.genesis.dag_genesis_block.getTimestamp();
+  bh->hash = dev::sha3(bh->ethereumRlp());
   return bh;
 }
 
+std::shared_ptr<BlockHeader> FinalChain::makeGenesisHeader(const h256& state_root) const {
+  auto header = std::make_shared<BlockHeader>();
+  header->timestamp = kConfig.genesis.dag_genesis_block.getTimestamp();
+  header->state_root = state_root;
+  header->gas_limit = kConfig.genesis.pbft.gas_limit;
+  header->hash = dev::sha3(header->ethereumRlp());
+  return header;
+}
+
 std::shared_ptr<const BlockHeader> FinalChain::getBlockHeader(EthBlockNumber n) const {
   if (auto raw = db_->lookup(n, DbStorage::Columns::final_chain_blk_by_number); !raw.empty()) {
     if (n == 0) {

libraries/core_libs/consensus/src/transaction/gas_pricer.cpp

@@ -1,6 +1,5 @@
 #include "transaction/gas_pricer.hpp"
 
-#include "final_chain/final_chain.hpp"
 #include "storage/storage.hpp"
 
 namespace taraxa {
@@ -28,7 +27,7 @@ u256 GasPricer::bid() const {
 
 void GasPricer::init(const std::shared_ptr<DbStorage>& db) {
   const auto last_blk_num =
-      db->lookup_int<EthBlockNumber>(final_chain::DBMetaKeys::LAST_NUMBER, DbStorage::Columns::final_chain_meta);
+      db->lookup_int<EthBlockNumber>(DBMetaKeys::LAST_NUMBER, DbStorage::Columns::final_chain_meta);
   if (!last_blk_num || *last_blk_num == 0) return;
   auto block_num = *last_blk_num;
 

libraries/core_libs/network/include/network/tarcap/taraxa_capability.hpp

@@ -121,8 +121,8 @@ class TaraxaCapability final : public dev::p2p::CapabilityFace {
   std::shared_ptr<threadpool::PacketsThreadPool> thread_pool_;
 
   // Last disconnect time and number of peers
-  std::chrono::_V2::system_clock::time_point last_ddos_disconnect_time_ = {};
-  std::chrono::_V2::system_clock::time_point queue_over_limit_start_time_ = {};
+  std::chrono::system_clock::time_point last_ddos_disconnect_time_ = {};
+  std::chrono::system_clock::time_point queue_over_limit_start_time_ = {};
   bool queue_over_limit_ = false;
   uint32_t last_disconnect_number_of_peers_ = 0;
 

libraries/core_libs/storage/include/storage/storage.hpp

@@ -52,6 +52,8 @@ enum PbftMgrStatus : uint8_t {
   NextVotedNullBlockHash,
 };
 
+enum class DBMetaKeys { LAST_NUMBER = 1 };
+
 class DbException : public std::exception {
  public:
   explicit DbException(const std::string& desc) : desc_(desc) {}

libraries/core_libs/storage/src/migration/final_chain_header.cpp

@@ -1,6 +1,5 @@
 #include "storage/migration/final_chain_header.hpp"
 
-#include "common/thread_pool.hpp"
 #include "final_chain/data.hpp"
 
 namespace taraxa::storage::migration {
@@ -11,51 +10,43 @@ std::string FinalChainHeader::id() { return "FinalChainHeader"; }
 
 uint32_t FinalChainHeader::dbVersion() { return 1; }
 
-struct OldHeader : final_chain::BlockHeaderData {
-  Address author;
-  EthBlockNumber number = 0;
-  uint64_t gas_limit = 0;
-  uint64_t timestamp = 0;
-  bytes extra_data;
-
+struct OldHeader : final_chain::BlockHeader {
   RLP_FIELDS_DEFINE_INPLACE(hash, parent_hash, author, state_root, transactions_root, receipts_root, log_bloom, number,
                             gas_limit, gas_used, timestamp, total_reward, extra_data)
 };
 
 void FinalChainHeader::migrate(logger::Logger& log) {
-  auto it = db_->getColumnIterator(DbStorage::Columns::final_chain_blk_by_number);
+  auto orig_col = DbStorage::Columns::final_chain_blk_by_number;
+  auto copied_col = db_->copyColumn(db_->handle(orig_col), orig_col.name() + "-copy");
+
+  if (copied_col == nullptr) {
+    LOG(log) << "Migration " << id() << " skipped: Unable to copy " << orig_col.name() << " column";
+    return;
+  }
+
+  auto it = db_->getColumnIterator(copied_col.get());
   it->SeekToFirst();
   if (!it->Valid()) {
     LOG(log) << "No blocks to migrate";
     return;
   }
 
-  uint64_t start_period, end_period;
-  memcpy(&start_period, it->key().data(), sizeof(uint64_t));
-
-  it->SeekToLast();
-  if (!it->Valid()) {
-    it->Prev();
-  }
-  memcpy(&end_period, it->key().data(), sizeof(uint64_t));
-  util::ThreadPool executor{std::thread::hardware_concurrency()};
-
-  const auto diff = (end_period - start_period) ? (end_period - start_period) : 1;
-  uint64_t curr_progress = 0;
-  std::cout << "Migrating " << diff << " blocks" << std::endl;
-  std::cout << "Start period: " << start_period << ", end period: " << end_period << std::endl;
-  // Get and save data in new format for all blocks
-  it->SeekToFirst();
+  uint64_t batch_size = 500000000;
   for (; it->Valid(); it->Next()) {
     uint64_t period;
     memcpy(&period, it->key().data(), sizeof(uint64_t));
     std::string raw = it->value().ToString();
-    executor.post([this, period, raw = std::move(raw), &copied_col]() {
-      auto header = std::make_shared<OldHeader>();
-      header->rlp(dev::RLP(raw));
-      auto newBytes = header->serializeForDB();
-      db_->insert(copied_col.get(), period, newBytes);
-    });
+    auto header = std::make_shared<OldHeader>();
+    header->rlp(dev::RLP(raw));
+    auto newBytes = header->serializeForDB();
+    db_->insert(batch_, copied_col.get(), period, newBytes);
+    if (batch_.GetDataSize() > batch_size) {
+      db_->commitWriteBatch(batch_);
+    }
   }
+  // commit the left over batch
+  db_->commitWriteBatch(batch_);
+
+  db_->replaceColumn(orig_col, std::move(copied_col));
 }
 }  // namespace taraxa::storage::migration