Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: improve prune #2465

Merged
merged 1 commit into from
May 4, 2023
Merged

chore: improve prune #2465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions libraries/core_libs/consensus/include/final_chain/state_api.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,7 @@ class StateAPI {
const RewardsStats& rewards_stats = {});
void transition_state_commit();
void create_snapshot(PbftPeriod period);
void prune(const dev::h256& state_root_to_keep, const std::vector<dev::h256>& state_root_to_prune,
EthBlockNumber blk_num);
void prune(const std::vector<dev::h256>& state_root_to_keep, EthBlockNumber blk_num);

// DPOS
uint64_t dpos_eligible_total_vote_count(EthBlockNumber blk_num) const;
Expand Down
3 changes: 3 additions & 0 deletions libraries/core_libs/consensus/src/dag/dag_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,8 @@ void DagManager::clearLightNodeHistory() {
// Actual history size will be between 100% and 110% of light_node_history_ to avoid deleting on every period
if (((period_ % (std::max(light_node_history_ / 10, (uint64_t)1)) == 0)) && period_ > light_node_history_ &&
dag_expiry_level_ > max_levels_per_period_ + 1) {
// This will happen at most once a day so log a silent log
LOG(log_si_) << "Clear light node history";
const auto proposal_period = db_->getProposalPeriodForDagLevel(dag_expiry_level_ - max_levels_per_period_ - 1);
assert(proposal_period);

Expand All @@ -299,6 +301,7 @@ void DagManager::clearLightNodeHistory() {
<< " *proposal_period " << *proposal_period;
LOG(log_tr_) << "Delete period history from: " << start << " to " << end;
db_->clearPeriodDataHistory(end);
LOG(log_si_) << "Clear light node history completed";
}
}

Expand Down
20 changes: 13 additions & 7 deletions libraries/core_libs/consensus/src/final_chain/final_chain.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,11 @@ class FinalChainImpl final : public FinalChain {
}

delegation_delay_ = config.genesis.state.dpos.delegation_delay;
if (config.db_config.prune_state_db && last_blk_num.has_value() && *last_blk_num > kLightNodeHistory) {
const auto kPruneblocksToKeep = kDagExpiryLevelLimit + kMaxLevelsPerPeriod + 1;
if ((config.db_config.prune_state_db || kLightNode) && last_blk_num.has_value() &&
*last_blk_num > kPruneblocksToKeep) {
LOG(log_si_) << "Pruning state db, this might take several minutes";
prune(*last_blk_num - kLightNodeHistory);
prune(*last_blk_num - kPruneblocksToKeep);
LOG(log_si_) << "Pruning state db complete";
}
}
Expand Down Expand Up @@ -246,13 +248,17 @@ class FinalChainImpl final : public FinalChain {
}

void prune(EthBlockNumber blk_n) override {
const auto last_block_to_keep = get_block_header(blk_n);
LOG(log_nf_) << "Pruning data older than " << blk_n;
auto last_block_to_keep = get_block_header(blk_n);
if (last_block_to_keep) {
std::vector<dev::h256> state_root_to_prune;
LOG(log_nf_) << "Pruning data older than " << blk_n;
auto block_to_keep = last_block_to_keep;
std::vector<dev::h256> state_root_to_keep;
while (block_to_keep) {
state_root_to_keep.push_back(block_to_keep->state_root);
block_to_keep = get_block_header(block_to_keep->number + 1);
}
auto block_to_prune = get_block_header(last_block_to_keep->number - 1);
while (block_to_prune && block_to_prune->number > 0) {
state_root_to_prune.push_back(block_to_prune->state_root);
db_->remove(DB::Columns::final_chain_blk_by_number, block_to_prune->number);
db_->remove(DB::Columns::final_chain_blk_hash_by_number, block_to_prune->number);
db_->remove(DB::Columns::final_chain_blk_number_by_hash, block_to_prune->hash);
Expand All @@ -263,7 +269,7 @@ class FinalChainImpl final : public FinalChain {
db_->compactColumn(DB::Columns::final_chain_blk_hash_by_number);
db_->compactColumn(DB::Columns::final_chain_blk_number_by_hash);

state_api_.prune(last_block_to_keep->state_root, state_root_to_prune, last_block_to_keep->number);
state_api_.prune(state_root_to_keep, last_block_to_keep->number);
}
}

Expand Down
5 changes: 2 additions & 3 deletions libraries/core_libs/consensus/src/final_chain/state_api.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,8 @@ void StateAPI::create_snapshot(PbftPeriod period) {
err_h.check();
}

void StateAPI::prune(const dev::h256& state_root_to_keep, const std::vector<dev::h256>& state_root_to_prune,
EthBlockNumber blk_num) {
return c_method_args_rlp<taraxa_evm_state_api_prune>(this_c_, state_root_to_keep, state_root_to_prune, blk_num);
void StateAPI::prune(const std::vector<dev::h256>& state_root_to_keep, EthBlockNumber blk_num) {
return c_method_args_rlp<taraxa_evm_state_api_prune>(this_c_, state_root_to_keep, blk_num);
}

uint64_t StateAPI::dpos_eligible_total_vote_count(EthBlockNumber blk_num) const {
Expand Down
2 changes: 1 addition & 1 deletion submodules/taraxa-evm
Open in desktop
Submodule taraxa-evm updated 5 files
+3 −4 taraxa/C/state.go
+5 −9 taraxa/state/state_db/block_reader.go
+82 −94 taraxa/state/state_db_rocksdb/db.go
+12 −50 taraxa/trie/reader.go
+2 −1 taraxa/trie/writer.go
24 changes: 3 additions & 21 deletions tests/full_node_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -685,31 +685,13 @@ TEST_F(FullNodeTest, sync_five_nodes) {

// Prune state_db of one node
auto prune_node = nodes[nodes.size() - 1];
const uint32_t min_blocks_to_prune = 50;
const uint32_t min_blocks_to_prune = 30;
// This ensures that we never prune blocks that are over proposal period
ASSERT_HAPPENS({20s, 100ms}, [&](auto &ctx) {
const auto max_level = prune_node->getDagManager()->getMaxLevel();
const auto proposal_period = prune_node->getDB()->getProposalPeriodForDagLevel(max_level);
ASSERT_TRUE(proposal_period.has_value());
context.dummy_transaction();
WAIT_EXPECT_TRUE(ctx, ((*proposal_period) > min_blocks_to_prune))
ASSERT_HAPPENS({40s, 100ms}, [&](auto &ctx) {
WAIT_EXPECT_TRUE(ctx, (prune_node->getPbftChain()->getPbftChainSize() > min_blocks_to_prune + kMaxLevelsPerPeriod))
});
prune_node->getFinalChain()->prune(min_blocks_to_prune);
context.assert_balances_synced();

// transfer some coins to pruned node ...
context.coin_transfer(0, prune_node->getAddress(), init_bal, false);
context.wait_all_transactions_known();

std::cout << "Waiting until transaction is executed" << std::endl;
auto trx_cnt = context.getIssuedTrxCount();
ASSERT_HAPPENS({20s, 500ms}, [&](auto &ctx) {
for (size_t i = 0; i < nodes.size(); ++i)
WAIT_EXPECT_EQ(ctx, nodes[i]->getDB()->getNumTransactionExecuted(), trx_cnt)
});

// Check balances after prune";
context.assert_balances_synced();
}

TEST_F(FullNodeTest, insert_anchor_and_compute_order) {
Expand Down