Skip to content

Commit

Permalink
Merge pull request #2465 from Taraxa-project/prune_replace_column
Browse files Browse the repository at this point in the history
chore: improve prune
  • Loading branch information
mfrankovi authored May 4, 2023
2 parents 63e07e3 + db9c16c commit 38821db
Show file tree
Hide file tree
Showing 6 changed files with 23 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,7 @@ class StateAPI {
const RewardsStats& rewards_stats = {});
void transition_state_commit();
void create_snapshot(PbftPeriod period);
void prune(const dev::h256& state_root_to_keep, const std::vector<dev::h256>& state_root_to_prune,
EthBlockNumber blk_num);
void prune(const std::vector<dev::h256>& state_root_to_keep, EthBlockNumber blk_num);

// DPOS
uint64_t dpos_eligible_total_vote_count(EthBlockNumber blk_num) const;
Expand Down
3 changes: 3 additions & 0 deletions libraries/core_libs/consensus/src/dag/dag_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,8 @@ void DagManager::clearLightNodeHistory() {
// Actual history size will be between 100% and 110% of light_node_history_ to avoid deleting on every period
if (((period_ % (std::max(light_node_history_ / 10, (uint64_t)1)) == 0)) && period_ > light_node_history_ &&
dag_expiry_level_ > max_levels_per_period_ + 1) {
// This will happen at most once a day so log a silent log
LOG(log_si_) << "Clear light node history";
const auto proposal_period = db_->getProposalPeriodForDagLevel(dag_expiry_level_ - max_levels_per_period_ - 1);
assert(proposal_period);

Expand All @@ -299,6 +301,7 @@ void DagManager::clearLightNodeHistory() {
<< " *proposal_period " << *proposal_period;
LOG(log_tr_) << "Delete period history from: " << start << " to " << end;
db_->clearPeriodDataHistory(end);
LOG(log_si_) << "Clear light node history completed";
}
}

Expand Down
20 changes: 13 additions & 7 deletions libraries/core_libs/consensus/src/final_chain/final_chain.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,11 @@ class FinalChainImpl final : public FinalChain {
}

delegation_delay_ = config.genesis.state.dpos.delegation_delay;
if (config.db_config.prune_state_db && last_blk_num.has_value() && *last_blk_num > kLightNodeHistory) {
const auto kPruneblocksToKeep = kDagExpiryLevelLimit + kMaxLevelsPerPeriod + 1;
if ((config.db_config.prune_state_db || kLightNode) && last_blk_num.has_value() &&
*last_blk_num > kPruneblocksToKeep) {
LOG(log_si_) << "Pruning state db, this might take several minutes";
prune(*last_blk_num - kLightNodeHistory);
prune(*last_blk_num - kPruneblocksToKeep);
LOG(log_si_) << "Pruning state db complete";
}
}
Expand Down Expand Up @@ -246,13 +248,17 @@ class FinalChainImpl final : public FinalChain {
}

void prune(EthBlockNumber blk_n) override {
const auto last_block_to_keep = get_block_header(blk_n);
LOG(log_nf_) << "Pruning data older than " << blk_n;
auto last_block_to_keep = get_block_header(blk_n);
if (last_block_to_keep) {
std::vector<dev::h256> state_root_to_prune;
LOG(log_nf_) << "Pruning data older than " << blk_n;
auto block_to_keep = last_block_to_keep;
std::vector<dev::h256> state_root_to_keep;
while (block_to_keep) {
state_root_to_keep.push_back(block_to_keep->state_root);
block_to_keep = get_block_header(block_to_keep->number + 1);
}
auto block_to_prune = get_block_header(last_block_to_keep->number - 1);
while (block_to_prune && block_to_prune->number > 0) {
state_root_to_prune.push_back(block_to_prune->state_root);
db_->remove(DB::Columns::final_chain_blk_by_number, block_to_prune->number);
db_->remove(DB::Columns::final_chain_blk_hash_by_number, block_to_prune->number);
db_->remove(DB::Columns::final_chain_blk_number_by_hash, block_to_prune->hash);
Expand All @@ -263,7 +269,7 @@ class FinalChainImpl final : public FinalChain {
db_->compactColumn(DB::Columns::final_chain_blk_hash_by_number);
db_->compactColumn(DB::Columns::final_chain_blk_number_by_hash);

state_api_.prune(last_block_to_keep->state_root, state_root_to_prune, last_block_to_keep->number);
state_api_.prune(state_root_to_keep, last_block_to_keep->number);
}
}

Expand Down
5 changes: 2 additions & 3 deletions libraries/core_libs/consensus/src/final_chain/state_api.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,8 @@ void StateAPI::create_snapshot(PbftPeriod period) {
err_h.check();
}

void StateAPI::prune(const dev::h256& state_root_to_keep, const std::vector<dev::h256>& state_root_to_prune,
EthBlockNumber blk_num) {
return c_method_args_rlp<taraxa_evm_state_api_prune>(this_c_, state_root_to_keep, state_root_to_prune, blk_num);
void StateAPI::prune(const std::vector<dev::h256>& state_root_to_keep, EthBlockNumber blk_num) {
return c_method_args_rlp<taraxa_evm_state_api_prune>(this_c_, state_root_to_keep, blk_num);
}

uint64_t StateAPI::dpos_eligible_total_vote_count(EthBlockNumber blk_num) const {
Expand Down
24 changes: 3 additions & 21 deletions tests/full_node_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -685,31 +685,13 @@ TEST_F(FullNodeTest, sync_five_nodes) {

// Prune state_db of one node
auto prune_node = nodes[nodes.size() - 1];
const uint32_t min_blocks_to_prune = 50;
const uint32_t min_blocks_to_prune = 30;
// This ensures that we never prune blocks that are over proposal period
ASSERT_HAPPENS({20s, 100ms}, [&](auto &ctx) {
const auto max_level = prune_node->getDagManager()->getMaxLevel();
const auto proposal_period = prune_node->getDB()->getProposalPeriodForDagLevel(max_level);
ASSERT_TRUE(proposal_period.has_value());
context.dummy_transaction();
WAIT_EXPECT_TRUE(ctx, ((*proposal_period) > min_blocks_to_prune))
ASSERT_HAPPENS({40s, 100ms}, [&](auto &ctx) {
WAIT_EXPECT_TRUE(ctx, (prune_node->getPbftChain()->getPbftChainSize() > min_blocks_to_prune + kMaxLevelsPerPeriod))
});
prune_node->getFinalChain()->prune(min_blocks_to_prune);
context.assert_balances_synced();

// transfer some coins to pruned node ...
context.coin_transfer(0, prune_node->getAddress(), init_bal, false);
context.wait_all_transactions_known();

std::cout << "Waiting until transaction is executed" << std::endl;
auto trx_cnt = context.getIssuedTrxCount();
ASSERT_HAPPENS({20s, 500ms}, [&](auto &ctx) {
for (size_t i = 0; i < nodes.size(); ++i)
WAIT_EXPECT_EQ(ctx, nodes[i]->getDB()->getNumTransactionExecuted(), trx_cnt)
});

// Check balances after prune";
context.assert_balances_synced();
}

TEST_F(FullNodeTest, insert_anchor_and_compute_order) {
Expand Down

0 comments on commit 38821db

Please sign in to comment.