From e56e36043319dcbd50b3029cde3028ba3f43e2e3 Mon Sep 17 00:00:00 2001 From: SW van Heerden Date: Mon, 12 Jul 2021 16:38:20 +0200 Subject: [PATCH] fix prune mode --- base_layer/core/src/base_node/proto/rpc.proto | 2 +- base_layer/core/src/base_node/proto/rpc.rs | 4 +- .../horizon_state_synchronization.rs | 4 +- base_layer/core/src/chain_storage/async_db.rs | 4 +- .../src/chain_storage/blockchain_database.rs | 11 ++- .../core/src/chain_storage/db_transaction.rs | 8 +- .../core/src/chain_storage/lmdb_db/lmdb_db.rs | 81 ++++++++++++------- .../core/src/chain_storage/lmdb_db/mod.rs | 2 +- .../core/src/chain_storage/pruned_output.rs | 2 +- base_layer/core/src/proto/block.proto | 2 +- base_layer/core/src/proto/block.rs | 6 +- 11 files changed, 77 insertions(+), 49 deletions(-) diff --git a/base_layer/core/src/base_node/proto/rpc.proto b/base_layer/core/src/base_node/proto/rpc.proto index 66a44247bb..968693ff8e 100644 --- a/base_layer/core/src/base_node/proto/rpc.proto +++ b/base_layer/core/src/base_node/proto/rpc.proto @@ -74,5 +74,5 @@ message SyncUtxo { message PrunedOutput { bytes hash = 1; - bytes rangeproof_hash = 2; + bytes witness_hash = 2; } diff --git a/base_layer/core/src/base_node/proto/rpc.rs b/base_layer/core/src/base_node/proto/rpc.rs index ccd3db5b73..e28355aeb2 100644 --- a/base_layer/core/src/base_node/proto/rpc.rs +++ b/base_layer/core/src/base_node/proto/rpc.rs @@ -36,11 +36,11 @@ impl From for proto::SyncUtxo { match output { PrunedOutput::Pruned { output_hash, - range_proof_hash, + witness_hash, } => proto::SyncUtxo { utxo: Some(proto::sync_utxo::Utxo::PrunedOutput(proto::PrunedOutput { hash: output_hash, - rangeproof_hash: range_proof_hash, + witness_hash, })), }, PrunedOutput::NotPruned { output } => proto::SyncUtxo { diff --git a/base_layer/core/src/base_node/state_machine_service/states/horizon_state_sync/horizon_state_synchronization.rs b/base_layer/core/src/base_node/state_machine_service/states/horizon_state_sync/horizon_state_synchronization.rs index 15a5b3005b..3a6a16b483 100644 --- a/base_layer/core/src/base_node/state_machine_service/states/horizon_state_sync/horizon_state_synchronization.rs +++ b/base_layer/core/src/base_node/state_machine_service/states/horizon_state_sync/horizon_state_synchronization.rs @@ -376,10 +376,10 @@ impl<'a, B: BlockchainBackend + 'static> HorizonStateSynchronization<'a, B> { ); height_txo_counter += 1; output_hashes.push(utxo.hash.clone()); - witness_hashes.push(utxo.rangeproof_hash.clone()); + witness_hashes.push(utxo.witness_hash.clone()); txn.insert_pruned_output_via_horizon_sync( utxo.hash, - utxo.rangeproof_hash, + utxo.witness_hash, current_header.hash().clone(), current_header.height(), u32::try_from(mmr_position)?, diff --git a/base_layer/core/src/chain_storage/async_db.rs b/base_layer/core/src/chain_storage/async_db.rs index b808976e80..fe298a71d4 100644 --- a/base_layer/core/src/chain_storage/async_db.rs +++ b/base_layer/core/src/chain_storage/async_db.rs @@ -284,13 +284,13 @@ impl<'a, B: BlockchainBackend + 'static> AsyncDbTransaction<'a, B> { pub fn insert_pruned_output_via_horizon_sync( &mut self, output_hash: HashOutput, - proof_hash: HashOutput, + witness_hash: HashOutput, header_hash: HashOutput, header_height: u64, mmr_position: u32, ) -> &mut Self { self.transaction - .insert_pruned_utxo(output_hash, proof_hash, header_hash, header_height, mmr_position); + .insert_pruned_utxo(output_hash, witness_hash, header_hash, header_height, mmr_position); self } diff --git a/base_layer/core/src/chain_storage/blockchain_database.rs b/base_layer/core/src/chain_storage/blockchain_database.rs index 52f51504b3..452021d58b 100644 --- a/base_layer/core/src/chain_storage/blockchain_database.rs +++ b/base_layer/core/src/chain_storage/blockchain_database.rs @@ -1138,9 +1138,9 @@ fn fetch_block(db: &T, height: u64) -> Result { - pruned.push((output_hash, range_proof_hash)); + pruned.push((output_hash, witness_hash)); }, PrunedOutput::NotPruned { output } => unpruned.push(output), } @@ -1844,6 +1844,13 @@ fn prune_database_if_needed( let db_height = metadata.height_of_longest_chain(); let abs_pruning_horizon = db_height.saturating_sub(pruning_horizon); + debug!( + target: LOG_TARGET, + "Current pruned height is: {}, pruning horizon is: {}, while the pruning interval is: {}", + metadata.pruned_height(), + abs_pruning_horizon, + pruning_interval, + ); if metadata.pruned_height() < abs_pruning_horizon.saturating_sub(pruning_interval) { let last_pruned = metadata.pruned_height(); info!( diff --git a/base_layer/core/src/chain_storage/db_transaction.rs b/base_layer/core/src/chain_storage/db_transaction.rs index 92e189ce11..c8bb8b959f 100644 --- a/base_layer/core/src/chain_storage/db_transaction.rs +++ b/base_layer/core/src/chain_storage/db_transaction.rs @@ -130,7 +130,7 @@ impl DbTransaction { pub fn insert_pruned_utxo( &mut self, output_hash: HashOutput, - proof_hash: HashOutput, + witness_hash: HashOutput, header_hash: HashOutput, header_height: u64, mmr_leaf_index: u32, @@ -139,7 +139,7 @@ impl DbTransaction { header_hash, header_height, output_hash, - proof_hash, + witness_hash, mmr_position: mmr_leaf_index, }); self @@ -297,7 +297,7 @@ pub enum WriteOperation { header_hash: HashOutput, header_height: u64, output_hash: HashOutput, - proof_hash: HashOutput, + witness_hash: HashOutput, mmr_position: u32, }, DeleteHeader(u64), @@ -410,7 +410,7 @@ impl fmt::Display for WriteOperation { header_hash: _, header_height: _, output_hash: _, - proof_hash: _, + witness_hash: _, mmr_position: _, } => write!(f, "Insert pruned output"), UpdateDeletedBlockAccumulatedDataWithDiff { diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 933318b0e1..f3f9e123d5 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -96,6 +96,24 @@ type DatabaseRef = Arc>; pub const LOG_TARGET: &str = "c::cs::lmdb_db::lmdb_db"; +struct OutputKey { + header_hash: HashOutput, + mmr_position: u32, +} + +impl OutputKey { + pub fn new(header_hash: HashOutput, mmr_position: u32) -> OutputKey { + OutputKey { + header_hash, + mmr_position, + } + } + + pub fn get_key(&self) -> String { + format!("{}-{:010}", self.header_hash.to_hex(), self.mmr_position) + } +} + /// This is a lmdb-based blockchain database for persistent storage of the chain state. pub struct LMDBDatabase { env: Arc, @@ -196,7 +214,7 @@ impl LMDBDatabase { header_hash, header_height, output_hash, - proof_hash, + witness_hash, mmr_position, } => { self.insert_pruned_output( @@ -204,7 +222,7 @@ impl LMDBDatabase { header_hash, header_height, output_hash, - proof_hash, + witness_hash, mmr_position, )?; }, @@ -323,13 +341,18 @@ impl LMDBDatabase { fn prune_output( &self, txn: &WriteTransaction<'_>, - key: &str, + key: &OutputKey, ) -> Result, ChainStorageError> { - let mut output: TransactionOutputRowData = - lmdb_get(txn, &self.utxos_db, key).or_not_found("TransactionOutput", "key", key.to_string())?; + let key = key.get_key(); + let key_string = key.as_str(); + let mut output: TransactionOutputRowData = lmdb_get(txn, &self.utxos_db, key_string).or_not_found( + "TransactionOutput", + "key", + key_string.to_string(), + )?; let result = output.output.take(); // output.output is None - lmdb_replace(txn, &self.utxos_db, key, &output)?; + lmdb_replace(txn, &self.utxos_db, key_string, &output)?; Ok(result) } @@ -342,25 +365,28 @@ impl LMDBDatabase { mmr_position: u32, ) -> Result<(), ChainStorageError> { let output_hash = output.hash(); - let proof_hash = output.proof.hash(); - let key = format!("{}-{:010}", header_hash.to_hex(), mmr_position,); + let witness_hash = output.witness_hash(); + + let key = OutputKey::new(header_hash.clone(), mmr_position); + let key_string = key.get_key(); + lmdb_insert( txn, &*self.txos_hash_to_index_db, output_hash.as_slice(), - &(mmr_position, key.clone()), + &(mmr_position, key_string.clone()), "txos_hash_to_index_db", )?; lmdb_insert( txn, &*self.utxos_db, - key.as_str(), + key_string.as_str(), &TransactionOutputRowData { output: Some(output), header_hash, mmr_position, hash: output_hash, - range_proof_hash: proof_hash, + witness_hash, mined_height: header_height, }, "utxos_db", @@ -373,39 +399,34 @@ impl LMDBDatabase { header_hash: HashOutput, header_height: u64, output_hash: HashOutput, - proof_hash: HashOutput, + witness_hash: HashOutput, mmr_position: u32, ) -> Result<(), ChainStorageError> { - if !lmdb_exists(txn, &self.headers_db, header_hash.as_slice())? { + if !lmdb_exists(txn, &self.block_hashes_db, header_hash.as_slice())? { return Err(ChainStorageError::InvalidOperation(format!( "Unable to insert pruned output because header {} does not exist", - header_hash.to_hex() + header_hash.to_hex(), ))); } - let key = format!( - "{}-{:010}-{}-{}", - header_hash.to_hex(), - mmr_position, - output_hash.to_hex(), - proof_hash.to_hex() - ); + let key = OutputKey::new(header_hash.clone(), mmr_position); + let key_string = key.get_key(); lmdb_insert( txn, &*self.txos_hash_to_index_db, output_hash.as_slice(), - &(mmr_position, key.clone()), + &(mmr_position, key_string.clone()), "txos_hash_to_index_db", )?; lmdb_insert( txn, &*self.utxos_db, - key.as_str(), + key_string.as_str(), &TransactionOutputRowData { output: None, header_hash, mmr_position, hash: output_hash, - range_proof_hash: proof_hash, + witness_hash, mined_height: header_height, }, "utxos_db", @@ -974,11 +995,11 @@ impl LMDBDatabase { let (_height, hash) = lmdb_first_after::<_, (u64, Vec)>( &write_txn, &self.output_mmr_size_index, - &(pos as u64).to_be_bytes(), + &((pos + 1) as u64).to_be_bytes(), ) .or_not_found("BlockHeader", "mmr_position", pos.to_string())?; - let key = format!("{}-{:010}", hash.to_hex(), pos); - debug!(target: LOG_TARGET, "Pruning output: {}", key); + let key = OutputKey::new(hash, pos); + debug!(target: LOG_TARGET, "Pruning output: {}", key.get_key()); self.prune_output(&write_txn, &key)?; } @@ -1538,7 +1559,7 @@ impl BlockchainBackend for LMDBDatabase { if deleted.contains(row.mmr_position) { return PrunedOutput::Pruned { output_hash: row.hash, - range_proof_hash: row.range_proof_hash, + witness_hash: row.witness_hash, }; } if let Some(output) = row.output { @@ -1546,7 +1567,7 @@ impl BlockchainBackend for LMDBDatabase { } else { PrunedOutput::Pruned { output_hash: row.hash, - range_proof_hash: row.range_proof_hash, + witness_hash: row.witness_hash, } } }), @@ -1625,7 +1646,7 @@ impl BlockchainBackend for LMDBDatabase { Some(o) => PrunedOutput::NotPruned { output: o }, None => PrunedOutput::Pruned { output_hash: f.hash, - range_proof_hash: f.range_proof_hash, + witness_hash: f.witness_hash, }, }) .collect(), diff --git a/base_layer/core/src/chain_storage/lmdb_db/mod.rs b/base_layer/core/src/chain_storage/lmdb_db/mod.rs index 49e3faf407..42ab93ef6e 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/mod.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/mod.rs @@ -56,7 +56,7 @@ pub(crate) struct TransactionOutputRowData { pub header_hash: HashOutput, pub mmr_position: u32, pub hash: HashOutput, - pub range_proof_hash: HashOutput, + pub witness_hash: HashOutput, pub mined_height: u64, } diff --git a/base_layer/core/src/chain_storage/pruned_output.rs b/base_layer/core/src/chain_storage/pruned_output.rs index 3e3628a42d..f49acf36ff 100644 --- a/base_layer/core/src/chain_storage/pruned_output.rs +++ b/base_layer/core/src/chain_storage/pruned_output.rs @@ -25,7 +25,7 @@ use crate::transactions::{transaction::TransactionOutput, types::HashOutput}; pub enum PrunedOutput { Pruned { output_hash: HashOutput, - range_proof_hash: HashOutput, + witness_hash: HashOutput, }, NotPruned { output: TransactionOutput, diff --git a/base_layer/core/src/proto/block.proto b/base_layer/core/src/proto/block.proto index ae6df42fd5..8f5edd5b12 100644 --- a/base_layer/core/src/proto/block.proto +++ b/base_layer/core/src/proto/block.proto @@ -72,7 +72,7 @@ message HistoricalBlock { Block block = 3; BlockHeaderAccumulatedData accumulated_data = 4; repeated bytes pruned_output_hashes = 5; - repeated bytes pruned_proof_hashes = 6; + repeated bytes pruned_witness_hash = 6; uint64 pruned_input_count = 7; } diff --git a/base_layer/core/src/proto/block.rs b/base_layer/core/src/proto/block.rs index 4c433c33cc..94a2f7fd20 100644 --- a/base_layer/core/src/proto/block.rs +++ b/base_layer/core/src/proto/block.rs @@ -79,7 +79,7 @@ impl TryFrom for HistoricalBlock { let pruned = historical_block .pruned_output_hashes .into_iter() - .zip(historical_block.pruned_proof_hashes) + .zip(historical_block.pruned_witness_hash) .collect(); Ok(HistoricalBlock::new( @@ -95,14 +95,14 @@ impl TryFrom for HistoricalBlock { impl From for proto::HistoricalBlock { fn from(block: HistoricalBlock) -> Self { let pruned_output_hashes = block.pruned_outputs().iter().map(|x| x.0.clone()).collect(); - let pruned_proof_hashes = block.pruned_outputs().iter().map(|x| x.1.clone()).collect(); + let pruned_witness_hash = block.pruned_outputs().iter().map(|x| x.1.clone()).collect(); let (block, accumulated_data, confirmations, pruned_input_count) = block.dissolve(); Self { confirmations, accumulated_data: Some(accumulated_data.into()), block: Some(block.into()), pruned_output_hashes, - pruned_proof_hashes, + pruned_witness_hash, pruned_input_count, } }