diff --git a/Cargo.lock b/Cargo.lock index 9dfd279170..188850e8e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5060,7 +5060,7 @@ dependencies = [ [[package]] name = "tari_wallet_ffi" -version = "0.17.3" +version = "0.17.4" dependencies = [ "chrono", "env_logger 0.7.1", diff --git a/applications/tari_console_wallet/src/automation/commands.rs b/applications/tari_console_wallet/src/automation/commands.rs index 608cc8a675..b5e5f334e1 100644 --- a/applications/tari_console_wallet/src/automation/commands.rs +++ b/applications/tari_console_wallet/src/automation/commands.rs @@ -499,14 +499,21 @@ pub async fn monitor_transactions( } } }, - TransactionEvent::TransactionMinedUnconfirmed(id, confirmations) if tx_ids.contains(id) => { + TransactionEvent::TransactionMinedUnconfirmed { + tx_id, + num_confirmations, + is_valid, + } if tx_ids.contains(tx_id) => { debug!( target: LOG_TARGET, - "tx mined unconfirmed event for tx_id: {}, confirmations: {}", *id, confirmations + "tx mined unconfirmed event for tx_id: {}, confirmations: {}, is_valid: {}", + *tx_id, + num_confirmations, + is_valid ); if wait_stage == TransactionStage::MinedUnconfirmed { results.push(SentTransaction { - id: *id, + id: *tx_id, stage: TransactionStage::MinedUnconfirmed, }); if results.len() == tx_ids.len() { @@ -514,11 +521,14 @@ pub async fn monitor_transactions( } } }, - TransactionEvent::TransactionMined(id) if tx_ids.contains(id) => { - debug!(target: LOG_TARGET, "tx mined confirmed event for tx_id: {}", *id); + TransactionEvent::TransactionMined { tx_id, is_valid } if tx_ids.contains(tx_id) => { + debug!( + target: LOG_TARGET, + "tx mined confirmed event for tx_id: {}, is_valid:{}", *tx_id, is_valid + ); if wait_stage == TransactionStage::Mined { results.push(SentTransaction { - id: *id, + id: *tx_id, stage: TransactionStage::Mined, }); if results.len() == tx_ids.len() { diff --git a/applications/tari_console_wallet/src/ui/state/wallet_event_monitor.rs b/applications/tari_console_wallet/src/ui/state/wallet_event_monitor.rs index 2e20999667..a7639720fc 100644 --- a/applications/tari_console_wallet/src/ui/state/wallet_event_monitor.rs +++ b/applications/tari_console_wallet/src/ui/state/wallet_event_monitor.rs @@ -71,12 +71,12 @@ impl WalletEventMonitor { self.trigger_tx_state_refresh(tx_id).await; notifier.transaction_received(tx_id); }, - TransactionEvent::TransactionMinedUnconfirmed(tx_id, confirmations) => { - self.trigger_confirmations_refresh(tx_id, confirmations).await; + TransactionEvent::TransactionMinedUnconfirmed{tx_id, num_confirmations, is_valid: _} => { + self.trigger_confirmations_refresh(tx_id, num_confirmations).await; self.trigger_tx_state_refresh(tx_id).await; - notifier.transaction_mined_unconfirmed(tx_id, confirmations); + notifier.transaction_mined_unconfirmed(tx_id, num_confirmations); }, - TransactionEvent::TransactionMined(tx_id) => { + TransactionEvent::TransactionMined{tx_id, is_valid: _} => { self.trigger_confirmations_cleanup(tx_id).await; self.trigger_tx_state_refresh(tx_id).await; notifier.transaction_mined(tx_id); diff --git a/applications/tari_console_wallet/src/ui/widgets/list_state.rs b/applications/tari_console_wallet/src/ui/widgets/list_state.rs index c95f259d80..f4c3026f5f 100644 --- a/applications/tari_console_wallet/src/ui/widgets/list_state.rs +++ b/applications/tari_console_wallet/src/ui/widgets/list_state.rs @@ -83,7 +83,7 @@ impl WindowedListState { let i = match self.selected { Some(i) => { if i >= self.num_items - 1 { - 0 + i } else { i + 1 } @@ -101,7 +101,7 @@ impl WindowedListState { let i = match self.selected { Some(i) => { if i == 0 { - self.num_items - 1 + i } else { i - 1 } diff --git a/applications/tari_explorer/routes/index.js b/applications/tari_explorer/routes/index.js index 2edce785d5..beddcbc78f 100644 --- a/applications/tari_explorer/routes/index.js +++ b/applications/tari_explorer/routes/index.js @@ -5,102 +5,112 @@ var router = express.Router() /* GET home page. */ router.get('/', async function (req, res, next) { - let client = createClient() - let from = parseInt(req.query.from || 0) - let limit = parseInt(req.query.limit || '20') + try { + let client = createClient() + let from = parseInt(req.query.from || 0) + let limit = parseInt(req.query.limit || '20') + + let tipInfo = await client.getTipInfo({}) + + console.log("Getting headers"); + // Algo split + let last100Headers = await client.listHeaders({ from_height: 0, num_headers: 101 }) + let monero = [0, 0, 0, 0] + let sha = [0, 0, 0, 0] + + console.log(last100Headers); + + for (let i = 0; i < last100Headers.length - 1; i++) { + let arr = last100Headers[i].pow.pow_algo === '0' ? monero : sha + if (i < 10) { + arr[0] += 1 + } + if (i < 20) { + arr[1] += 1 + } + if (i < 50) { + arr[2] += 1 + } + arr[3] += 1 - let tipInfo = await client.getTipInfo({}) - - // Algo split - let last100Headers = await client.listHeaders({ from_height: 0, num_headers: 101 }) - let monero = [0, 0, 0, 0] - let sha = [0, 0, 0, 0] - - for (let i = 0; i < last100Headers.length - 1; i++) { - let arr = last100Headers[i].pow.pow_algo === '0' ? monero : sha - if (i < 10) { - arr[0] += 1 } - if (i < 20) { - arr[1] += 1 + const algoSplit = { + monero10: monero[0], + monero20: monero[1], + monero50: monero[2], + monero100: monero[3], + sha10: sha[0], + sha20: sha[1], + sha50: sha[2], + sha100: sha[3] } - if (i < 50) { - arr[2] += 1 - } - arr[3] += 1 - - } - const algoSplit = { - monero10: monero[0], - monero20: monero[1], - monero50: monero[2], - monero100: monero[3], - sha10: sha[0], - sha20: sha[1], - sha50: sha[2], - sha100: sha[3] - } - + console.log(algoSplit); + // Get one more header than requested so we can work out the difference in MMR_size + let headers = await client.listHeaders({ from_height: from, num_headers: limit + 1 }) + for (var i = headers.length - 2; i >= 0; i--) { + headers[i].kernels = headers[i].kernel_mmr_size - headers[i + 1].kernel_mmr_size + headers[i].outputs = headers[i].output_mmr_size - headers[i + 1].output_mmr_size + } + let lastHeader = headers[headers.length - 1] + if (lastHeader.height === '0') { + // If the block is the genesis block, then the MMR sizes are the values to use + lastHeader.kernels = lastHeader.kernel_mmr_size + lastHeader.outputs = lastHeader.output_mmr_size + } else { + // Otherwise remove the last one, as we don't want to show it + headers.splice(headers.length - 1, 1) + } + // console.log(headers); + let firstHeight = parseInt(headers[0].height || '0') - // Get one more header than requested so we can work out the difference in MMR_size - let headers = await client.listHeaders({ from_height: from, num_headers: limit + 1 }) - for (var i = headers.length - 2; i >= 0; i--) { - headers[i].kernels = headers[i].kernel_mmr_size - headers[i + 1].kernel_mmr_size - headers[i].outputs = headers[i].output_mmr_size - headers[i + 1].output_mmr_size - } - let lastHeader = headers[headers.length - 1] - if (lastHeader.height === '0') { - // If the block is the genesis block, then the MMR sizes are the values to use - lastHeader.kernels = lastHeader.kernel_mmr_size - lastHeader.outputs = lastHeader.output_mmr_size - } else { - // Otherwise remove the last one, as we don't want to show it - headers.splice(headers.length - 1, 1) - } - - // console.log(headers); - let firstHeight = parseInt(headers[0].height || '0') - - // -- mempool - let mempool = await client.getMempoolTransactions({}) + // -- mempool + let mempool = await client.getMempoolTransactions({}) - for (let i = 0; i < mempool.length; i++) { - let sum = 0 - for (let j = 0; j < mempool[i].transaction.body.kernels.length; j++) { - sum += parseInt(mempool[i].transaction.body.kernels[j].fee) + for (let i = 0; i < mempool.length; i++) { + let sum = 0 + for (let j = 0; j < mempool[i].transaction.body.kernels.length; j++) { + sum += parseInt(mempool[i].transaction.body.kernels[j].fee) + } + mempool[i].transaction.body.total_fees = sum } - mempool[i].transaction.body.total_fees = sum + res.render('index', { + title: 'Blocks', + tipInfo: tipInfo, + mempool: mempool, + headers: headers, + pows: { '0': 'Monero', '2': 'SHA' }, + nextPage: firstHeight - limit, + prevPage: firstHeight + limit, + limit: limit, + from: from, + algoSplit: algoSplit, + blockTimes: getBlockTimes(last100Headers), + moneroTimes: getBlockTimes(last100Headers, "0"), + shaTimes: getBlockTimes(last100Headers, "1") + }) + + } catch (error) { + res.status(500) + res.render('error', { error: error }) } - console.log(getBlockTimes(last100Headers, "0")) - res.render('index', { - title: 'Blocks', - tipInfo: tipInfo, - mempool: mempool, - headers: headers, - pows: { '0': 'Monero', '2': 'SHA' }, - nextPage: firstHeight - limit, - prevPage: firstHeight + limit, - limit: limit, - from: from, - algoSplit: algoSplit, - blockTimes: getBlockTimes(last100Headers), - moneroTimes: getBlockTimes(last100Headers, "0"), - shaTimes: getBlockTimes(last100Headers, "2") - }) }) function getBlockTimes(last100Headers, algo) { let blocktimes = [] let i = 0 - if (algo === '0' || algo === '2') { - while (last100Headers[i].pow.pow_algo !== algo) { + if (algo === '0' || algo === '1') { + while (i < last100Headers.length && last100Headers[i].pow.pow_algo !== algo) { i++; blocktimes.push(0) } } + if (i >= last100Headers.length) { + // This happens if there are no blocks for a specific algorithm in last100headers + return blocktimes; + } let lastBlockTime = parseInt(last100Headers[i].timestamp.seconds); i++; while (i< last100Headers.length && blocktimes.length < 60) { diff --git a/base_layer/core/src/base_node/proto/wallet_rpc.proto b/base_layer/core/src/base_node/proto/wallet_rpc.proto index d67750f9b5..fc241cd974 100644 --- a/base_layer/core/src/base_node/proto/wallet_rpc.proto +++ b/base_layer/core/src/base_node/proto/wallet_rpc.proto @@ -43,11 +43,14 @@ message TxQueryBatchResponse { TxLocation location = 2; google.protobuf.BytesValue block_hash = 3; uint64 confirmations = 4; + uint64 block_height = 5; } message TxQueryBatchResponses { repeated TxQueryBatchResponse responses = 1; bool is_synced = 2; + google.protobuf.BytesValue tip_hash = 3; + uint64 height_of_longest_chain = 4; } @@ -60,6 +63,38 @@ message FetchUtxosResponse { bool is_synced = 2; } + +message QueryDeletedRequest{ + repeated uint64 mmr_positions = 1; + google.protobuf.BytesValue chain_must_include_header = 2; +} + +message QueryDeletedResponse { + repeated uint64 deleted_positions = 1; + repeated uint64 not_deleted_positions = 2; + bytes best_block = 3; + uint64 height_of_longest_chain = 4; +} + +message UtxoQueryRequest{ + repeated bytes output_hashes =1; +} + +message UtxoQueryResponses { + repeated UtxoQueryResponse responses =1; + bytes best_block = 3; + uint64 height_of_longest_chain = 4; +} + +message UtxoQueryResponse { + tari.types.TransactionOutput output = 1; + uint64 mmr_position = 2; + uint64 mined_height =3; + bytes mined_in_block = 4; + bytes output_hash = 5; + +} + message TipInfoResponse { ChainMetadata metadata = 1; bool is_synced = 2; diff --git a/base_layer/core/src/base_node/proto/wallet_rpc.rs b/base_layer/core/src/base_node/proto/wallet_rpc.rs index 94f2f2d7f6..3ca41c15d7 100644 --- a/base_layer/core/src/base_node/proto/wallet_rpc.rs +++ b/base_layer/core/src/base_node/proto/wallet_rpc.rs @@ -134,6 +134,7 @@ pub struct TxQueryBatchResponse { pub location: TxLocation, pub block_hash: Option, pub confirmations: u64, + pub block_height: u64, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -224,6 +225,7 @@ impl TryFrom for TxQueryBatchResponse { .ok_or_else(|| "Invalid or unrecognised `TxLocation` enum".to_string())?, )?, block_hash: proto_response.block_hash, + block_height: proto_response.block_height, confirmations: proto_response.confirmations, }) } diff --git a/base_layer/core/src/base_node/rpc/mod.rs b/base_layer/core/src/base_node/rpc/mod.rs index 44cdefdbc3..16d35c0efa 100644 --- a/base_layer/core/src/base_node/rpc/mod.rs +++ b/base_layer/core/src/base_node/rpc/mod.rs @@ -48,6 +48,7 @@ use crate::{ }, }; +use crate::proto::base_node::{QueryDeletedRequest, QueryDeletedResponse, UtxoQueryRequest, UtxoQueryResponses}; use tari_comms::protocol::rpc::{Request, Response, RpcStatus}; use tari_comms_rpc_macros::tari_rpc; @@ -79,6 +80,15 @@ pub trait BaseNodeWalletService: Send + Sync + 'static { #[rpc(method = 6)] async fn get_header(&self, request: Request) -> Result, RpcStatus>; + + #[rpc(method = 7)] + async fn utxo_query(&self, request: Request) -> Result, RpcStatus>; + + #[rpc(method = 8)] + async fn query_deleted( + &self, + request: Request, + ) -> Result, RpcStatus>; } #[cfg(feature = "base_node")] diff --git a/base_layer/core/src/base_node/rpc/service.rs b/base_layer/core/src/base_node/rpc/service.rs index c50600ea9c..db482f123c 100644 --- a/base_layer/core/src/base_node/rpc/service.rs +++ b/base_layer/core/src/base_node/rpc/service.rs @@ -19,16 +19,17 @@ // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - use crate::{ base_node::{rpc::BaseNodeWalletService, state_machine_service::states::StateInfo, StateMachineHandle}, - chain_storage::{async_db::AsyncBlockchainDb, BlockchainBackend, PrunedOutput}, + chain_storage::{async_db::AsyncBlockchainDb, BlockchainBackend, PrunedOutput, UtxoMinedInfo}, mempool::{service::MempoolHandle, TxStorageResponse}, proto, proto::{ base_node::{ FetchMatchingUtxos, FetchUtxosResponse, + QueryDeletedRequest, + QueryDeletedResponse, Signatures as SignaturesProto, TipInfoResponse, TxLocation, @@ -37,12 +38,15 @@ use crate::{ TxQueryResponse, TxSubmissionRejectionReason, TxSubmissionResponse, + UtxoQueryRequest, + UtxoQueryResponse, + UtxoQueryResponses, }, types::{Signature as SignatureProto, Transaction as TransactionProto}, }, transactions::{transaction::Transaction, types::Signature}, }; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use tari_comms::protocol::rpc::{Request, Response, RpcStatus}; const LOG_TARGET: &str = "c::base_node::rpc"; @@ -260,6 +264,12 @@ impl BaseNodeWalletService for BaseNodeWalletRpc let mut responses: Vec = Vec::new(); + let metadata = self + .db + .get_chain_metadata() + .await + .map_err(RpcStatus::log_internal_error(LOG_TARGET))?; + for sig in message.sigs { let signature = Signature::try_from(sig).map_err(|_| RpcStatus::bad_request("Signature was invalid"))?; let response: TxQueryResponse = self.fetch_kernel(signature.clone()).await?; @@ -268,9 +278,15 @@ impl BaseNodeWalletService for BaseNodeWalletRpc location: response.location, block_hash: response.block_hash, confirmations: response.confirmations, + block_height: response.height_of_longest_chain - response.confirmations, }); } - Ok(Response::new(TxQueryBatchResponses { responses, is_synced })) + Ok(Response::new(TxQueryBatchResponses { + responses, + is_synced, + tip_hash: Some(metadata.best_block().clone()), + height_of_longest_chain: metadata.height_of_longest_chain(), + })) } async fn fetch_matching_utxos( @@ -309,6 +325,116 @@ impl BaseNodeWalletService for BaseNodeWalletRpc })) } + async fn utxo_query(&self, request: Request) -> Result, RpcStatus> { + let message = request.into_message(); + let db = self.db(); + let mut res = Vec::with_capacity(message.output_hashes.len()); + for UtxoMinedInfo { + output, + mmr_position, + mined_height: height, + header_hash, + } in (db + .fetch_utxos_and_mined_info(message.output_hashes) + .await + .map_err(RpcStatus::log_internal_error(LOG_TARGET))?) + .into_iter() + .flatten() + { + res.push((output, mmr_position, height, header_hash)); + } + + let metadata = self + .db + .get_chain_metadata() + .await + .map_err(RpcStatus::log_internal_error(LOG_TARGET))?; + + // let deleted = self + // .db + // .fetch_complete_deleted_bitmap_at(metadata.best_block().clone()) + // .await + // .map_err(RpcStatus::log_internal_error(LOG_TARGET))? + // .into_bytes(); + Ok(Response::new(UtxoQueryResponses { + height_of_longest_chain: metadata.height_of_longest_chain(), + best_block: metadata.best_block().clone(), + responses: res + .into_iter() + .map( + |(output, mmr_position, mined_height, mined_in_block)| UtxoQueryResponse { + mmr_position: mmr_position.into(), + mined_height, + mined_in_block, + output_hash: output.hash(), + output: match output { + PrunedOutput::Pruned { .. } => None, + PrunedOutput::NotPruned { output } => Some(output.into()), + }, + }, + ) + .collect(), + })) + } + + /// Currently the wallet cannot use the deleted bitmap because it can't compile croaring + /// at some point in the future, it might be better to send the wallet the actual bitmap so + /// it can check itself + async fn query_deleted( + &self, + request: Request, + ) -> Result, RpcStatus> { + let message = request.into_message(); + + if let Some(chain_must_include_header) = message.chain_must_include_header { + if self + .db + .fetch_header_by_block_hash(chain_must_include_header) + .await + .map_err(RpcStatus::log_internal_error(LOG_TARGET))? + .is_none() + { + return Err(RpcStatus::not_found( + "Chain does not include header. It might have been reorged out", + )); + } + } + + let metadata = self + .db + .get_chain_metadata() + .await + .map_err(RpcStatus::log_internal_error(LOG_TARGET))?; + + let deleted_bitmap = self + .db + .fetch_complete_deleted_bitmap_at(metadata.best_block().clone()) + .await + .map_err(RpcStatus::log_internal_error(LOG_TARGET))?; + + let mut deleted_positions = vec![]; + let mut not_deleted_positions = vec![]; + + for position in message.mmr_positions { + if position > u32::MAX as u64 { + // TODO: in future, bitmap may support higher than u32 + return Err(RpcStatus::bad_request("position must fit into a u32")); + } + let pos = position.try_into().unwrap(); + if deleted_bitmap.bitmap().contains(pos) { + deleted_positions.push(position); + } else { + not_deleted_positions.push(position); + } + } + Ok(Response::new(QueryDeletedResponse { + height_of_longest_chain: metadata.height_of_longest_chain(), + best_block: metadata.best_block().clone(), + deleted_positions, + not_deleted_positions, + })) + } + async fn get_tip_info(&self, _request: Request<()>) -> Result, RpcStatus> { let state_machine = self.state_machine(); let status_watch = state_machine.get_status_info_watch(); diff --git a/base_layer/core/src/chain_storage/accumulated_data.rs b/base_layer/core/src/chain_storage/accumulated_data.rs index 40f0bac01b..52e456830a 100644 --- a/base_layer/core/src/chain_storage/accumulated_data.rs +++ b/base_layer/core/src/chain_storage/accumulated_data.rs @@ -229,6 +229,10 @@ impl CompleteDeletedBitmap { pub fn dissolve(self) -> (Bitmap, u64, HashOutput) { (self.deleted, self.height, self.hash) } + + pub fn into_bytes(self) -> Vec { + self.deleted.serialize() + } } pub struct BlockHeaderAccumulatedDataBuilder<'a> { diff --git a/base_layer/core/src/chain_storage/async_db.rs b/base_layer/core/src/chain_storage/async_db.rs index 2ca767024d..e2bce858b8 100644 --- a/base_layer/core/src/chain_storage/async_db.rs +++ b/base_layer/core/src/chain_storage/async_db.rs @@ -24,6 +24,7 @@ use crate::{ blocks::{Block, BlockHeader, NewBlockTemplate}, chain_storage::{ accumulated_data::BlockHeaderAccumulatedData, + utxo_mined_info::UtxoMinedInfo, BlockAccumulatedData, BlockAddResult, BlockchainBackend, @@ -143,6 +144,8 @@ impl AsyncBlockchainDb { make_async_fn!(fetch_utxos(hashes: Vec) -> Vec>, "fetch_utxos"); + make_async_fn!(fetch_utxos_and_mined_info(hashes: Vec) -> Vec>, "fetch_utxos_and_mined_info"); + make_async_fn!(fetch_utxos_by_mmr_position(start: u64, end: u64, deleted: Arc) -> (Vec, Bitmap), "fetch_utxos_by_mmr_position"); //---------------------------------- Kernel --------------------------------------------// diff --git a/base_layer/core/src/chain_storage/blockchain_backend.rs b/base_layer/core/src/chain_storage/blockchain_backend.rs index f5d3b6ad36..c803081d6c 100644 --- a/base_layer/core/src/chain_storage/blockchain_backend.rs +++ b/base_layer/core/src/chain_storage/blockchain_backend.rs @@ -3,6 +3,7 @@ use crate::{ chain_storage::{ accumulated_data::DeletedBitmap, pruned_output::PrunedOutput, + utxo_mined_info::UtxoMinedInfo, BlockAccumulatedData, BlockHeaderAccumulatedData, ChainBlock, @@ -103,7 +104,7 @@ pub trait BlockchainBackend: Send + Sync { ) -> Result<(Vec, Bitmap), ChainStorageError>; /// Fetch a specific output. Returns the output and the leaf index in the output MMR - fn fetch_output(&self, output_hash: &HashOutput) -> Result, ChainStorageError>; + fn fetch_output(&self, output_hash: &HashOutput) -> Result, ChainStorageError>; /// Returns the unspent TransactionOutput output that matches the given commitment if it exists in the current UTXO /// set, otherwise None is returned. diff --git a/base_layer/core/src/chain_storage/blockchain_database.rs b/base_layer/core/src/chain_storage/blockchain_database.rs index 769ac64cfd..cadaead025 100644 --- a/base_layer/core/src/chain_storage/blockchain_database.rs +++ b/base_layer/core/src/chain_storage/blockchain_database.rs @@ -31,6 +31,7 @@ use crate::{ db_transaction::{DbKey, DbTransaction, DbValue}, error::ChainStorageError, pruned_output::PrunedOutput, + utxo_mined_info::UtxoMinedInfo, BlockAddResult, BlockchainBackend, ChainBlock, @@ -286,7 +287,7 @@ where B: BlockchainBackend // Fetch the utxo pub fn fetch_utxo(&self, hash: HashOutput) -> Result, ChainStorageError> { let db = self.db_read_access()?; - Ok(db.fetch_output(&hash)?.map(|(out, _index, _)| out)) + Ok(db.fetch_output(&hash)?.map(|mined_info| mined_info.output)) } pub fn fetch_unspent_output_by_commitment( @@ -307,7 +308,22 @@ where B: BlockchainBackend let mut result = Vec::with_capacity(hashes.len()); for hash in hashes { let output = db.fetch_output(&hash)?; - result.push(output.map(|(out, mmr_index, _)| (out, deleted.bitmap().contains(mmr_index)))); + result + .push(output.map(|mined_info| (mined_info.output, deleted.bitmap().contains(mined_info.mmr_position)))); + } + Ok(result) + } + + pub fn fetch_utxos_and_mined_info( + &self, + hashes: Vec, + ) -> Result>, ChainStorageError> { + let db = self.db_read_access()?; + + let mut result = Vec::with_capacity(hashes.len()); + for hash in hashes { + let output = db.fetch_output(&hash)?; + result.push(output); } Ok(result) } @@ -1262,19 +1278,10 @@ fn fetch_block_with_utxo( db: &T, commitment: Commitment, ) -> Result, ChainStorageError> { - match db.fetch_output(&commitment.to_vec()) { - Ok(output) => match output { - Some((_output, leaf, _height)) => { - let header = db.fetch_header_containing_utxo_mmr(leaf as u64)?; - fetch_block_by_hash(db, header.hash().to_owned()) - }, - None => Ok(None), - }, - Err(_) => Err(ChainStorageError::ValueNotFound { - entity: "Output", - field: "Commitment", - value: commitment.to_hex(), - }), + let output = db.fetch_output(&commitment.to_vec())?; + match output { + Some(mined_info) => fetch_block_by_hash(db, mined_info.header_hash), + None => Ok(None), } } diff --git a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs index 7bb5908911..8c9c708e58 100644 --- a/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs +++ b/base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs @@ -65,6 +65,7 @@ use crate::{ LMDB_DB_UTXO_COMMITMENT_INDEX, LMDB_DB_UTXO_MMR_SIZE_INDEX, }, + utxo_mined_info::UtxoMinedInfo, BlockchainBackend, ChainBlock, ChainHeader, @@ -1721,7 +1722,7 @@ impl BlockchainBackend for LMDBDatabase { Ok((result, difference_bitmap)) } - fn fetch_output(&self, output_hash: &HashOutput) -> Result, ChainStorageError> { + fn fetch_output(&self, output_hash: &HashOutput) -> Result, ChainStorageError> { debug!(target: LOG_TARGET, "Fetch output: {}", output_hash.to_hex()); let txn = self.read_transaction()?; if let Some((index, key)) = @@ -1739,27 +1740,31 @@ impl BlockchainBackend for LMDBDatabase { output: Some(o), mmr_position, mined_height, + header_hash, .. - }) => Ok(Some(( - PrunedOutput::NotPruned { output: o }, + }) => Ok(Some(UtxoMinedInfo { + output: PrunedOutput::NotPruned { output: o }, mmr_position, mined_height, - ))), + header_hash, + })), Some(TransactionOutputRowData { output: None, mmr_position, mined_height, hash, witness_hash, + header_hash, .. - }) => Ok(Some(( - PrunedOutput::Pruned { + }) => Ok(Some(UtxoMinedInfo { + output: PrunedOutput::Pruned { output_hash: hash, witness_hash, }, mmr_position, mined_height, - ))), + header_hash, + })), _ => Ok(None), } } else { diff --git a/base_layer/core/src/chain_storage/mod.rs b/base_layer/core/src/chain_storage/mod.rs index 11755caccb..d77103fce8 100644 --- a/base_layer/core/src/chain_storage/mod.rs +++ b/base_layer/core/src/chain_storage/mod.rs @@ -93,4 +93,7 @@ pub use lmdb_db::{ }; mod target_difficulties; +mod utxo_mined_info; +pub use utxo_mined_info::*; + pub use target_difficulties::TargetDifficulties; diff --git a/base_layer/core/src/chain_storage/pruned_output.rs b/base_layer/core/src/chain_storage/pruned_output.rs index 957c0e8c86..f5137d6da3 100644 --- a/base_layer/core/src/chain_storage/pruned_output.rs +++ b/base_layer/core/src/chain_storage/pruned_output.rs @@ -20,6 +20,7 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. use crate::transactions::{transaction::TransactionOutput, types::HashOutput}; +use tari_crypto::tari_utilities::Hashable; #[allow(clippy::large_enum_variant)] #[derive(Debug, PartialEq)] @@ -37,4 +38,14 @@ impl PrunedOutput { pub fn is_pruned(&self) -> bool { matches!(self, PrunedOutput::Pruned { .. }) } + + pub fn hash(&self) -> Vec { + match self { + PrunedOutput::Pruned { + output_hash, + witness_hash: _, + } => output_hash.clone(), + PrunedOutput::NotPruned { output } => output.hash(), + } + } } diff --git a/base_layer/core/src/chain_storage/utxo_mined_info.rs b/base_layer/core/src/chain_storage/utxo_mined_info.rs new file mode 100644 index 0000000000..1105ceb981 --- /dev/null +++ b/base_layer/core/src/chain_storage/utxo_mined_info.rs @@ -0,0 +1,31 @@ +// Copyright 2021. The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use crate::chain_storage::PrunedOutput; +use tari_common_types::types::BlockHash; + +pub struct UtxoMinedInfo { + pub output: PrunedOutput, + pub mmr_position: u32, + pub mined_height: u64, + pub header_hash: BlockHash, +} diff --git a/base_layer/core/src/test_helpers/blockchain.rs b/base_layer/core/src/test_helpers/blockchain.rs index e871ed10c9..21a0c9e5e1 100644 --- a/base_layer/core/src/test_helpers/blockchain.rs +++ b/base_layer/core/src/test_helpers/blockchain.rs @@ -19,7 +19,6 @@ // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - use crate::{ blocks::{genesis_block::get_weatherwax_genesis_block, Block, BlockHeader}, chain_storage::{ @@ -40,6 +39,7 @@ use crate::{ LMDBDatabase, MmrTree, PrunedOutput, + UtxoMinedInfo, Validators, }, consensus::{chain_strength_comparer::ChainStrengthComparerBuilder, ConsensusConstantsBuilder, ConsensusManager}, @@ -244,7 +244,7 @@ impl BlockchainBackend for TempDatabase { self.db.fetch_utxos_by_mmr_position(start, end, deleted) } - fn fetch_output(&self, output_hash: &HashOutput) -> Result, ChainStorageError> { + fn fetch_output(&self, output_hash: &HashOutput) -> Result, ChainStorageError> { self.db.fetch_output(output_hash) } diff --git a/base_layer/wallet/migrations/2019-10-30-084148_output_manager_service/up.sql b/base_layer/wallet/migrations/2019-10-30-084148_output_manager_service/up.sql index b03a9d1bda..35ec650f86 100644 --- a/base_layer/wallet/migrations/2019-10-30-084148_output_manager_service/up.sql +++ b/base_layer/wallet/migrations/2019-10-30-084148_output_manager_service/up.sql @@ -8,7 +8,7 @@ CREATE TABLE outputs ( ); CREATE TABLE pending_transaction_outputs ( - tx_id INTEGER PRIMARY KEY NOT NULL, + tx_id BIGINT PRIMARY KEY NOT NULL, short_term INTEGER NOT NULL, timestamp DATETIME NOT NULL ); @@ -19,4 +19,4 @@ CREATE TABLE key_manager_states ( branch_seed TEXT NOT NULL, primary_key_index INTEGER NOT NULL, timestamp DATETIME NOT NULL -); \ No newline at end of file +); diff --git a/base_layer/wallet/migrations/2019-11-20-090620_transaction_service/up.sql b/base_layer/wallet/migrations/2019-11-20-090620_transaction_service/up.sql index 29eb8b0935..e940facfa5 100644 --- a/base_layer/wallet/migrations/2019-11-20-090620_transaction_service/up.sql +++ b/base_layer/wallet/migrations/2019-11-20-090620_transaction_service/up.sql @@ -1,37 +1,37 @@ CREATE TABLE outbound_transactions ( - tx_id INTEGER PRIMARY KEY NOT NULL, + tx_id BIGINT PRIMARY KEY NOT NULL, destination_public_key BLOB NOT NULL, - amount INTEGER NOT NULL, - fee INTEGER NOT NULL, + amount BIGINT NOT NULL, + fee BIGINT NOT NULL, sender_protocol TEXT NOT NULL, message TEXT NOT NULL, timestamp DATETIME NOT NULL ); CREATE TABLE inbound_transactions ( - tx_id INTEGER PRIMARY KEY NOT NULL, + tx_id BIGINT PRIMARY KEY NOT NULL, source_public_key BLOB NOT NULL, - amount INTEGER NOT NULL, + amount BIGINT NOT NULL, receiver_protocol TEXT NOT NULL, message TEXT NOT NULL, timestamp DATETIME NOT NULL ); CREATE TABLE coinbase_transactions ( - tx_id INTEGER PRIMARY KEY NOT NULL, - amount INTEGER NOT NULL, + tx_id BIGINT PRIMARY KEY NOT NULL, + amount BIGINT NOT NULL, commitment BLOB NOT NULL, timestamp DATETIME NOT NULL ); CREATE TABLE completed_transactions ( - tx_id INTEGER PRIMARY KEY NOT NULL, + tx_id BIGINT PRIMARY KEY NOT NULL, source_public_key BLOB NOT NULL, destination_public_key BLOB NOT NULL, - amount INTEGER NOT NULL, - fee INTEGER NOT NULL, + amount BIGINT NOT NULL, + fee BIGINT NOT NULL, transaction_protocol TEXT NOT NULL, status INTEGER NOT NULL, message TEXT NOT NULL, timestamp DATETIME NOT NULL -); \ No newline at end of file +); diff --git a/base_layer/wallet/migrations/2020-07-20-084915_add_coinbase_handling/up.sql b/base_layer/wallet/migrations/2020-07-20-084915_add_coinbase_handling/up.sql index 14d4297d0c..99aebb2190 100644 --- a/base_layer/wallet/migrations/2020-07-20-084915_add_coinbase_handling/up.sql +++ b/base_layer/wallet/migrations/2020-07-20-084915_add_coinbase_handling/up.sql @@ -3,10 +3,10 @@ PRAGMA foreign_keys=off; ALTER TABLE key_manager_states RENAME TO key_manager_states_old; CREATE TABLE key_manager_states ( - id INTEGER PRIMARY KEY, + id INTEGER PRIMARY KEY NOT NULL, master_key BLOB NOT NULL, branch_seed TEXT NOT NULL, - primary_key_index INTEGER NOT NULL, + primary_key_index BIGINT NOT NULL, timestamp DATETIME NOT NULL ); INSERT INTO key_manager_states (id, master_key, branch_seed, primary_key_index, timestamp) @@ -15,6 +15,6 @@ SELECT id, master_seed, branch_seed, primary_key_index, timestamp DROP TABLE key_manager_states_old; PRAGMA foreign_keys=on; -ALTER TABLE pending_transaction_outputs ADD COLUMN coinbase_block_height INTEGER NULL; +ALTER TABLE pending_transaction_outputs ADD COLUMN coinbase_block_height BIGINT NULL; -ALTER TABLE completed_transactions ADD COLUMN coinbase_block_height INTEGER NULL; +ALTER TABLE completed_transactions ADD COLUMN coinbase_block_height BIGINT NULL; diff --git a/base_layer/wallet/migrations/2021-04-01-081220_add_transaction_confirmations/up.sql b/base_layer/wallet/migrations/2021-04-01-081220_add_transaction_confirmations/up.sql index 0f583618a9..c88bd991f1 100644 --- a/base_layer/wallet/migrations/2021-04-01-081220_add_transaction_confirmations/up.sql +++ b/base_layer/wallet/migrations/2021-04-01-081220_add_transaction_confirmations/up.sql @@ -1,2 +1,2 @@ ALTER TABLE completed_transactions - ADD COLUMN confirmations INTEGER NULL DEFAULT NULL; \ No newline at end of file + ADD COLUMN confirmations BIGINT NULL DEFAULT NULL; diff --git a/base_layer/wallet/migrations/2021-04-19-085137_add_mined_height_to_completed_transaction/up.sql b/base_layer/wallet/migrations/2021-04-19-085137_add_mined_height_to_completed_transaction/up.sql index 5ec7109251..393d990670 100644 --- a/base_layer/wallet/migrations/2021-04-19-085137_add_mined_height_to_completed_transaction/up.sql +++ b/base_layer/wallet/migrations/2021-04-19-085137_add_mined_height_to_completed_transaction/up.sql @@ -1,2 +1,2 @@ ALTER TABLE completed_transactions - ADD COLUMN mined_height INTEGER NULL; \ No newline at end of file + ADD COLUMN mined_height BIGINT NULL; diff --git a/base_layer/wallet/migrations/2021-07-05-13201407_metadata_signature/up.sql b/base_layer/wallet/migrations/2021-07-05-13201407_metadata_signature/up.sql index 50656c8686..1c8c892d0f 100644 --- a/base_layer/wallet/migrations/2021-07-05-13201407_metadata_signature/up.sql +++ b/base_layer/wallet/migrations/2021-07-05-13201407_metadata_signature/up.sql @@ -4,15 +4,15 @@ PRAGMA foreign_keys=off; DROP TABLE outputs; CREATE TABLE outputs ( - id INTEGER NOT NULL PRIMARY KEY, - commitment BLOB NOT NULL, + id INTEGER NOT NULL PRIMARY KEY, --auto inc, + commitment BLOB NULL, spending_key BLOB NOT NULL, - value INTEGER NOT NULL, + value BIGINT NOT NULL, flags INTEGER NOT NULL, - maturity INTEGER NOT NULL, + maturity bigint NOT NULL, status INTEGER NOT NULL, - tx_id INTEGER NULL, - hash BLOB NOT NULL, + tx_id bigint NULL, + hash BLOB NULL, script BLOB NOT NULL, input_data BLOB NOT NULL, script_private_key BLOB NOT NULL, diff --git a/base_layer/wallet/migrations/2021-07-28-120000_add_mined_in_block/down.sql b/base_layer/wallet/migrations/2021-07-28-120000_add_mined_in_block/down.sql new file mode 100644 index 0000000000..409150c9c7 --- /dev/null +++ b/base_layer/wallet/migrations/2021-07-28-120000_add_mined_in_block/down.sql @@ -0,0 +1,2 @@ + +-- not supported diff --git a/base_layer/wallet/migrations/2021-07-28-120000_add_mined_in_block/up.sql b/base_layer/wallet/migrations/2021-07-28-120000_add_mined_in_block/up.sql new file mode 100644 index 0000000000..eb3566b9ef --- /dev/null +++ b/base_layer/wallet/migrations/2021-07-28-120000_add_mined_in_block/up.sql @@ -0,0 +1,33 @@ +-- Copyright 2021. The Tari Project +-- +-- Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +-- following conditions are met: +-- +-- 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +-- disclaimer. +-- +-- 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +-- following disclaimer in the documentation and/or other materials provided with the distribution. +-- +-- 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +-- products derived from this software without specific prior written permission. +-- +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +-- INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +-- WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +-- USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +alter table completed_transactions +add mined_in_block blob null; + +alter table outputs +add mined_height unsigned bigint null; + +alter table outputs +add mined_in_block blob null; + +alter table outputs +add mined_mmr_position bigint null; diff --git a/base_layer/wallet/migrations/2021-08-03-123456_update_outputs_mined/down.sql b/base_layer/wallet/migrations/2021-08-03-123456_update_outputs_mined/down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/base_layer/wallet/migrations/2021-08-03-123456_update_outputs_mined/up.sql b/base_layer/wallet/migrations/2021-08-03-123456_update_outputs_mined/up.sql new file mode 100644 index 0000000000..d00905210d --- /dev/null +++ b/base_layer/wallet/migrations/2021-08-03-123456_update_outputs_mined/up.sql @@ -0,0 +1,29 @@ +-- Copyright 2021. The Tari Project +-- +-- Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +-- following conditions are met: +-- +-- 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +-- disclaimer. +-- +-- 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +-- following disclaimer in the documentation and/or other materials provided with the distribution. +-- +-- 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +-- products derived from this software without specific prior written permission. +-- +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +-- INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +-- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +-- WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +-- USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +alter table outputs add marked_deleted_at_height BigInt; +alter table outputs add marked_deleted_in_block blob; +alter table outputs add received_in_tx_id bigint; +alter table outputs add spent_in_tx_id bigint; +update outputs set received_in_tx_id = tx_id; + +-- TODO: drop outputs tx_id column diff --git a/base_layer/wallet/src/contacts_service/storage/sqlite_db.rs b/base_layer/wallet/src/contacts_service/storage/sqlite_db.rs index 8fc8234f56..4cfdb57747 100644 --- a/base_layer/wallet/src/contacts_service/storage/sqlite_db.rs +++ b/base_layer/wallet/src/contacts_service/storage/sqlite_db.rs @@ -27,6 +27,7 @@ use crate::{ }, schema::contacts, storage::sqlite_utilities::WalletDbConnection, + util::diesel_ext::ExpectedRowsExtension, }; use diesel::{prelude::*, result::Error as DieselError, SqliteConnection}; use std::convert::TryFrom; @@ -141,15 +142,10 @@ impl ContactSql { updated_contact: UpdateContact, conn: &SqliteConnection, ) -> Result { - let num_updated = diesel::update(contacts::table.filter(contacts::public_key.eq(&self.public_key))) + diesel::update(contacts::table.filter(contacts::public_key.eq(&self.public_key))) .set(updated_contact) - .execute(conn)?; - - if num_updated == 0 { - return Err(ContactsServiceStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + .execute(conn) + .num_rows_affected_or_not_found(1)?; ContactSql::find(&self.public_key, conn) } diff --git a/base_layer/wallet/src/lib.rs b/base_layer/wallet/src/lib.rs index bc0b4c1a04..683aee00f3 100644 --- a/base_layer/wallet/src/lib.rs +++ b/base_layer/wallet/src/lib.rs @@ -6,6 +6,10 @@ #![deny(unreachable_patterns)] #![deny(unknown_lints)] #![recursion_limit = "2048"] +// Some functions have a large amount of dependencies (e.g. services) and historically this warning +// has lead to bundling of dependencies into a resources struct, which is then overused and is the +// wrong abstraction +#![allow(clippy::too_many_arguments)] #[macro_use] mod macros; diff --git a/base_layer/wallet/src/output_manager_service/error.rs b/base_layer/wallet/src/output_manager_service/error.rs index 1dfd3e969e..1fc807a857 100644 --- a/base_layer/wallet/src/output_manager_service/error.rs +++ b/base_layer/wallet/src/output_manager_service/error.rs @@ -22,7 +22,7 @@ use crate::base_node_service::error::BaseNodeServiceError; use diesel::result::Error as DieselError; -use tari_comms::{peer_manager::node_id::NodeIdError, protocol::rpc::RpcError}; +use tari_comms::{connectivity::ConnectivityError, peer_manager::node_id::NodeIdError, protocol::rpc::RpcError}; use tari_comms_dht::outbound::DhtOutboundError; use tari_core::transactions::{ transaction::TransactionError, @@ -109,6 +109,13 @@ pub enum OutputManagerError { MasterSecretKeyMismatch, #[error("Private Key is not found in the current Key Chain")] KeyNotFoundInKeyChain, + #[error("Connectivity error: {source}")] + ConnectivityError { + #[from] + source: ConnectivityError, + }, + #[error("Invalid message received:{0}")] + InvalidMessageError(String), } #[derive(Debug, Error, PartialEq)] @@ -178,3 +185,16 @@ impl From for OutputManagerError { tspe.error } } + +pub trait OutputManagerProtocolErrorExt { + fn for_protocol(self, id: u64) -> Result; +} + +impl> OutputManagerProtocolErrorExt for Result { + fn for_protocol(self, id: u64) -> Result { + match self { + Ok(r) => Ok(r), + Err(e) => Err(OutputManagerProtocolError::new(id, e.into())), + } + } +} diff --git a/base_layer/wallet/src/output_manager_service/service.rs b/base_layer/wallet/src/output_manager_service/service.rs index c8946b9ff9..d9223cb501 100644 --- a/base_layer/wallet/src/output_manager_service/service.rs +++ b/base_layer/wallet/src/output_manager_service/service.rs @@ -32,7 +32,7 @@ use crate::{ database::{OutputManagerBackend, OutputManagerDatabase, PendingTransactionOutputs}, models::{DbUnblindedOutput, KnownOneSidedPaymentScript}, }, - tasks::{TxoValidationTask, TxoValidationType}, + tasks::{TxoValidationTaskV2, TxoValidationType}, MasterKeyManager, TxId, }, @@ -40,7 +40,6 @@ use crate::{ types::{HashDigest, ValidationRetryStrategy}, }; use blake2::Digest; -use chrono::Utc; use diesel::result::{DatabaseErrorKind, Error as DieselError}; use futures::{pin_mut, StreamExt}; use log::*; @@ -50,7 +49,7 @@ use std::{ collections::HashMap, fmt::{self, Display}, sync::Arc, - time::Duration, + time::{Duration, Instant}, }; use tari_comms::{ connectivity::ConnectivityRequester, @@ -85,7 +84,7 @@ use tari_crypto::{ }; use tari_service_framework::reply_channel; use tari_shutdown::ShutdownSignal; -use tokio::sync::broadcast; +use tokio::{sync::broadcast, time::interval_at}; const LOG_TARGET: &str = "wallet::output_manager_service"; const LOG_TARGET_STRESS: &str = "stress_test::output_manager_service"; @@ -163,10 +162,25 @@ where TBackend: OutputManagerBackend + 'static pin_mut!(request_stream); let mut shutdown = self.resources.shutdown_signal.clone(); - + // Should probably be block time or at least configurable + // Start a bit later so that the wallet can start up + let mut txo_validation_interval = interval_at( + (Instant::now() + Duration::from_secs(35)).into(), + Duration::from_secs(30), + ) + .fuse(); info!(target: LOG_TARGET, "Output Manager Service started"); loop { futures::select! { + // tx validation timer + _ = txo_validation_interval.select_next_some() => { + let _ =self + .validate_outputs(TxoValidationType::Unspent, ValidationRetryStrategy::Limited(0)).map_err(|e| { + warn!(target: LOG_TARGET, "Error validating txos: {:?}", e); + e + }); + }, + request_context = request_stream.select_next_some() => { trace!(target: LOG_TARGET, "Handling Service API Request"); let (request, reply_tx) = request_context.split(); @@ -357,24 +371,53 @@ where TBackend: OutputManagerBackend + 'static fn validate_outputs( &mut self, validation_type: TxoValidationType, - retry_strategy: ValidationRetryStrategy, + _retry_strategy: ValidationRetryStrategy, ) -> Result { match self.resources.base_node_public_key.as_ref() { None => Err(OutputManagerError::NoBaseNodeKeysProvided), Some(pk) => { let id = OsRng.next_u64(); - - let utxo_validation_task = TxoValidationTask::new( + // let utxo_validation_task = TxoValidationTask::new( + // id, + // validation_type, + // retry_strategy, + // self.resources.clone(), + // pk.clone(), + // self.base_node_update_publisher.subscribe(), + // ); + // + // tokio::spawn(async move { + // match utxo_validation_task.execute().await { + // Ok(id) => { + // info!( + // target: LOG_TARGET, + // "UTXO Validation Protocol (Id: {}) completed successfully", id + // ); + // }, + // Err(OutputManagerProtocolError { id, error }) => { + // warn!( + // target: LOG_TARGET, + // "Error completing UTXO Validation Protocol (Id: {}): {:?}", id, error + // ); + // }, + // } + // }); + + let utxo_validation = TxoValidationTaskV2::new( + pk.clone(), id, + 100, + self.resources.db.clone(), + self.resources.connectivity_manager.clone(), + self.resources.event_publisher.clone(), validation_type, - retry_strategy, - self.resources.clone(), - pk.clone(), - self.base_node_update_publisher.subscribe(), + self.resources.config.clone(), ); + let shutdown = self.resources.shutdown_signal.clone(); + tokio::spawn(async move { - match utxo_validation_task.execute().await { + match utxo_validation.execute(shutdown).await { Ok(id) => { info!( target: LOG_TARGET, @@ -389,7 +432,6 @@ where TBackend: OutputManagerBackend + 'static }, } }); - Ok(id) }, } @@ -908,23 +950,25 @@ where TBackend: OutputManagerBackend + 'static /// Restore the pending transaction encumberance and output for an inbound transaction that was previously /// cancelled. - async fn reinstate_cancelled_inbound_transaction(&mut self, tx_id: TxId) -> Result<(), OutputManagerError> { - self.resources.db.reinstate_inbound_output(tx_id).await?; - - self.resources - .db - .add_pending_transaction_outputs(PendingTransactionOutputs { - tx_id, - outputs_to_be_spent: Vec::new(), - outputs_to_be_received: Vec::new(), - timestamp: Utc::now().naive_utc(), - coinbase_block_height: None, - }) - .await?; - - self.confirm_encumberance(tx_id).await?; - - Ok(()) + async fn reinstate_cancelled_inbound_transaction(&mut self, _tx_id: TxId) -> Result<(), OutputManagerError> { + // TODO: is this still needed? + unimplemented!() + // self.resources.db.reinstate_inbound_output(tx_id).await?; + // + // self.resources + // .db + // .add_pending_transaction_outputs(PendingTransactionOutputs { + // tx_id, + // outputs_to_be_spent: Vec::new(), + // outputs_to_be_received: Vec::new(), + // timestamp: Utc::now().naive_utc(), + // coinbase_block_height: None, + // }) + // .await?; + // + // self.confirm_encumberance(tx_id).await?; + + // Ok(()) } /// Go through the pending transaction and if any have existed longer than the specified duration, cancel them diff --git a/base_layer/wallet/src/output_manager_service/storage/database.rs b/base_layer/wallet/src/output_manager_service/storage/database.rs index 52d552e016..f619fb9814 100644 --- a/base_layer/wallet/src/output_manager_service/storage/database.rs +++ b/base_layer/wallet/src/output_manager_service/storage/database.rs @@ -38,7 +38,7 @@ use std::{ use tari_core::transactions::{ tari_amount::MicroTari, transaction::TransactionOutput, - types::{BlindingFactor, Commitment, PrivateKey}, + types::{BlindingFactor, Commitment, HashOutput, PrivateKey}, }; const LOG_TARGET: &str = "wallet::output_manager_service::database"; @@ -50,12 +50,36 @@ const LOG_TARGET: &str = "wallet::output_manager_service::database"; pub trait OutputManagerBackend: Send + Sync + Clone { /// Retrieve the record associated with the provided DbKey fn fetch(&self, key: &DbKey) -> Result, OutputManagerStorageError>; + /// Retrieve outputs that have not been found in the block chain + fn fetch_unmined_spent_outputs(&self) -> Result, OutputManagerStorageError>; + /// Retrieve outputs that have not been found in the block chain + fn fetch_unmined_received_outputs(&self) -> Result, OutputManagerStorageError>; /// Modify the state the of the backend with a write operation fn write(&self, op: WriteOperation) -> Result, OutputManagerStorageError>; + + fn set_output_mined_height( + &self, + hash: Vec, + mined_height: u64, + mined_in_block: Vec, + mmr_position: u64, + ) -> Result<(), OutputManagerStorageError>; + + fn set_output_to_unmined(&self, hash: Vec) -> Result<(), OutputManagerStorageError>; + + fn mark_output_as_spent( + &self, + hash: Vec, + mark_deleted_at_height: u64, + mark_deleted_in_block: Vec, + ) -> Result<(), OutputManagerStorageError>; + + fn mark_output_as_unspent(&self, hash: Vec) -> Result<(), OutputManagerStorageError>; + /// This method is called when a pending transaction is to be confirmed. It must move the `outputs_to_be_spent` and /// `outputs_to_be_received` from a `PendingTransactionOutputs` record into the `unspent_outputs` and /// `spent_outputs` collections. - fn confirm_transaction(&self, tx_id: TxId) -> Result<(), OutputManagerStorageError>; + fn confirm_transaction_encumberance(&self, tx_id: TxId) -> Result<(), OutputManagerStorageError>; /// This method encumbers the specified outputs into a `PendingTransactionOutputs` record. This is a short term /// encumberance in case the app is closed or crashes before transaction neogtiation is complete. These will be /// cleared on startup of the service. @@ -102,6 +126,11 @@ pub trait OutputManagerBackend: Send + Sync + Clone { &self, commitment: &Commitment, ) -> Result; + + /// Get the output that was most recently mined, ordered descending by mined height + fn get_last_mined_output(&self) -> Result, OutputManagerStorageError>; + /// Get the output that was most recently spent, ordered descending by mined height + fn get_last_spent_output(&self) -> Result, OutputManagerStorageError>; } /// Holds the outputs that have been selected for a given pending transaction waiting for confirmation @@ -160,7 +189,6 @@ pub enum DbKeyValuePair { PendingTransactionOutputs(TxId, Box), KeyManagerState(KeyManagerState), KnownOneSidedPaymentScripts(KnownOneSidedPaymentScript), - UpdateOutputStatus(Commitment, OutputStatus), } pub enum WriteOperation { @@ -380,7 +408,7 @@ where T: OutputManagerBackend + 'static /// `spent_outputs` collections. pub async fn confirm_pending_transaction_outputs(&self, tx_id: TxId) -> Result<(), OutputManagerStorageError> { let db_clone = self.db.clone(); - tokio::task::spawn_blocking(move || db_clone.confirm_transaction(tx_id)) + tokio::task::spawn_blocking(move || db_clone.confirm_transaction_encumberance(tx_id)) .await .map_err(|err| OutputManagerStorageError::BlockingTaskSpawnError(err.to_string())) .and_then(|inner_result| inner_result) @@ -507,6 +535,22 @@ where T: OutputManagerBackend + 'static Ok(uo) } + pub async fn fetch_unmined_received_outputs(&self) -> Result, OutputManagerStorageError> { + let db_clone = self.db.clone(); + let utxos = tokio::task::spawn_blocking(move || db_clone.fetch_unmined_received_outputs()) + .await + .map_err(|err| OutputManagerStorageError::BlockingTaskSpawnError(err.to_string()))??; + Ok(utxos) + } + + pub async fn fetch_unmined_spent_outputs(&self) -> Result, OutputManagerStorageError> { + let db_clone = self.db.clone(); + let utxos = tokio::task::spawn_blocking(move || db_clone.fetch_unmined_spent_outputs()) + .await + .map_err(|err| OutputManagerStorageError::BlockingTaskSpawnError(err.to_string()))??; + Ok(utxos) + } + pub async fn fetch_all_pending_transaction_outputs( &self, ) -> Result, OutputManagerStorageError> { @@ -683,6 +727,14 @@ where T: OutputManagerBackend + 'static Ok(scripts) } + pub async fn get_last_mined_output(&self) -> Result, OutputManagerStorageError> { + self.db.get_last_mined_output() + } + + pub async fn get_last_spent_output(&self) -> Result, OutputManagerStorageError> { + self.db.get_last_spent_output() + } + pub async fn add_known_script( &self, known_script: KnownOneSidedPaymentScript, @@ -714,45 +766,48 @@ where T: OutputManagerBackend + 'static Ok(()) } - /// Check if a single cancelled inbound output exists that matches this TxID, if it does then return its status to - /// EncumberedToBeReceived - pub async fn reinstate_inbound_output(&self, tx_id: TxId) -> Result<(), OutputManagerStorageError> { - let db_clone = self.db.clone(); - let outputs = tokio::task::spawn_blocking(move || { - match db_clone.fetch(&DbKey::OutputsByTxIdAndStatus(tx_id, OutputStatus::CancelledInbound)) { - Ok(None) => Err(OutputManagerStorageError::ValueNotFound), - Ok(Some(DbValue::AnyOutputs(o))) => Ok(o), - Ok(Some(other)) => unexpected_result( - DbKey::OutputsByTxIdAndStatus(tx_id, OutputStatus::CancelledInbound), - other, - ), - Err(e) => log_error(DbKey::OutputsByTxIdAndStatus(tx_id, OutputStatus::CancelledInbound), e), - } - }) - .await - .map_err(|err| OutputManagerStorageError::BlockingTaskSpawnError(err.to_string())) - .and_then(|inner_result| inner_result)?; - - if outputs.len() != 1 { - return Err(OutputManagerStorageError::UnexpectedResult( - "There should be only 1 output for a cancelled inbound transaction but more were found".to_string(), - )); - } - let db_clone2 = self.db.clone(); - + pub async fn set_output_mined_height( + &self, + hash: HashOutput, + mined_height: u64, + mined_in_block: HashOutput, + mmr_position: u64, + ) -> Result<(), OutputManagerStorageError> { + let db = self.db.clone(); tokio::task::spawn_blocking(move || { - db_clone2.write(WriteOperation::Insert(DbKeyValuePair::UpdateOutputStatus( - outputs - .first() - .expect("Must be only one element in outputs") - .commitment - .clone(), - OutputStatus::EncumberedToBeReceived, - ))) + db.set_output_mined_height(hash, mined_height, mined_in_block, mmr_position) }) .await .map_err(|err| OutputManagerStorageError::BlockingTaskSpawnError(err.to_string()))??; + Ok(()) + } + + pub async fn set_output_as_unmined(&self, hash: HashOutput) -> Result<(), OutputManagerStorageError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || db.set_output_to_unmined(hash)) + .await + .map_err(|err| OutputManagerStorageError::BlockingTaskSpawnError(err.to_string()))??; + Ok(()) + } + pub async fn mark_output_as_spent( + &self, + hash: HashOutput, + mined_height: u64, + mined_in_block: HashOutput, + ) -> Result<(), OutputManagerStorageError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || db.mark_output_as_spent(hash, mined_height, mined_in_block)) + .await + .map_err(|err| OutputManagerStorageError::BlockingTaskSpawnError(err.to_string()))??; + Ok(()) + } + + pub async fn mark_output_as_unspent(&self, hash: HashOutput) -> Result<(), OutputManagerStorageError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || db.mark_output_as_unspent(hash)) + .await + .map_err(|err| OutputManagerStorageError::BlockingTaskSpawnError(err.to_string()))??; Ok(()) } } diff --git a/base_layer/wallet/src/output_manager_service/storage/models.rs b/base_layer/wallet/src/output_manager_service/storage/models.rs index dd36eb6934..e313180c68 100644 --- a/base_layer/wallet/src/output_manager_service/storage/models.rs +++ b/base_layer/wallet/src/output_manager_service/storage/models.rs @@ -22,6 +22,7 @@ use crate::output_manager_service::error::OutputManagerStorageError; use std::cmp::Ordering; +use tari_common_types::types::BlockHash; use tari_core::{ tari_utilities::hash::Hashable, transactions::{ @@ -37,6 +38,11 @@ pub struct DbUnblindedOutput { pub commitment: Commitment, pub unblinded_output: UnblindedOutput, pub hash: HashOutput, + pub mined_height: Option, + pub mined_in_block: Option, + pub mined_mmr_position: Option, + pub marked_deleted_at_height: Option, + pub marked_deleted_in_block: Option, } impl DbUnblindedOutput { @@ -49,6 +55,11 @@ impl DbUnblindedOutput { hash: tx_out.hash(), commitment: tx_out.commitment, unblinded_output: output, + mined_height: None, + mined_in_block: None, + mined_mmr_position: None, + marked_deleted_at_height: None, + marked_deleted_in_block: None, }) } @@ -62,6 +73,11 @@ impl DbUnblindedOutput { hash: tx_out.hash(), commitment: tx_out.commitment, unblinded_output: output, + mined_height: None, + mined_in_block: None, + mined_mmr_position: None, + marked_deleted_at_height: None, + marked_deleted_in_block: None, }) } } diff --git a/base_layer/wallet/src/output_manager_service/storage/sqlite_db.rs b/base_layer/wallet/src/output_manager_service/storage/sqlite_db.rs index 665c408bdf..6345fa8e17 100644 --- a/base_layer/wallet/src/output_manager_service/storage/sqlite_db.rs +++ b/base_layer/wallet/src/output_manager_service/storage/sqlite_db.rs @@ -39,7 +39,10 @@ use crate::{ }, schema::{key_manager_states, known_one_sided_payment_scripts, outputs, pending_transaction_outputs}, storage::sqlite_utilities::WalletDbConnection, - util::encryption::{decrypt_bytes_integral_nonce, encrypt_bytes_integral_nonce, Encryptable}, + util::{ + diesel_ext::ExpectedRowsExtension, + encryption::{decrypt_bytes_integral_nonce, encrypt_bytes_integral_nonce, Encryptable}, + }, }; use aes_gcm::{aead::Error as AeadError, Aes256Gcm, Error}; use chrono::{Duration as ChronoDuration, NaiveDateTime, Utc}; @@ -47,7 +50,7 @@ use diesel::{prelude::*, result::Error as DieselError, SqliteConnection}; use log::*; use std::{ collections::HashMap, - convert::TryFrom, + convert::{TryFrom, TryInto}, str::from_utf8, sync::{Arc, RwLock}, time::Duration, @@ -253,6 +256,10 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { Some(mut km) => { self.decrypt_if_necessary(&mut km)?; + // TODO: This is a problem because the keymanager state does not have an index + // meaning that update round trips to the database can't be found again. + // I would suggest changing this to a different pattern for retrieval, perhaps + // only returning the columns that are needed. Some(DbValue::KeyManagerState(KeyManagerState::try_from(km)?)) }, }, @@ -287,21 +294,47 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { Ok(result) } - #[allow(clippy::cognitive_complexity)] + fn fetch_unmined_spent_outputs(&self) -> Result, OutputManagerStorageError> { + let conn = self.database_connection.acquire_lock(); + let mut outputs = OutputSql::index_marked_deleted_in_block_is_null(&(*conn))?; + for output in outputs.iter_mut() { + self.decrypt_if_necessary(output)?; + } + + outputs + .into_iter() + .map(DbUnblindedOutput::try_from) + .collect::, _>>() + } + + fn fetch_unmined_received_outputs(&self) -> Result, OutputManagerStorageError> { + let conn = self.database_connection.acquire_lock(); + let mut outputs = OutputSql::index_mined_in_block_is_null(&(*conn))?; + for output in outputs.iter_mut() { + self.decrypt_if_necessary(output)?; + } + + outputs + .into_iter() + .map(DbUnblindedOutput::try_from) + .collect::, _>>() + } + fn write(&self, op: WriteOperation) -> Result, OutputManagerStorageError> { let conn = self.database_connection.acquire_lock(); match op { WriteOperation::Insert(kvp) => match kvp { - DbKeyValuePair::SpentOutput(c, o) => { - if OutputSql::find_by_commitment_and_cancelled(&c.to_vec(), false, &(*conn)).is_ok() { - return Err(OutputManagerStorageError::DuplicateOutput); - } - let mut new_output = NewOutputSql::new(*o, OutputStatus::Spent, None)?; - - self.encrypt_if_necessary(&mut new_output)?; - - new_output.commit(&(*conn))? + DbKeyValuePair::SpentOutput(_c, _o) => { + unimplemented!("Deprecated") + // if OutputSql::find_by_commitment_and_cancelled(&c.to_vec(), false, &(*conn)).is_ok() { + // return Err(OutputManagerStorageError::DuplicateOutput); + // } + // let mut new_output = NewOutputSql::new(*o, OutputStatus::Spent, None)?; + // + // self.encrypt_if_necessary(&mut new_output)?; + // + // new_output.commit(&(*conn))? }, DbKeyValuePair::UnspentOutput(c, o) => { if OutputSql::find_by_commitment_and_cancelled(&c.to_vec(), false, &(*conn)).is_ok() { @@ -320,6 +353,7 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { new_output.commit(&(*conn))? }, DbKeyValuePair::PendingTransactionOutputs(tx_id, p) => { + // TODO: Used only by coinbases, so should be renamed if PendingTransactionOutputSql::find(tx_id, &(*conn)).is_ok() { return Err(OutputManagerStorageError::DuplicateTransaction); } @@ -331,10 +365,11 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { p.coinbase_block_height.map(|h| h as i64), ) .commit(&(*conn))?; - for o in p.outputs_to_be_spent { - let mut new_output = NewOutputSql::new(o, OutputStatus::EncumberedToBeSpent, Some(p.tx_id))?; - self.encrypt_if_necessary(&mut new_output)?; - new_output.commit(&(*conn))?; + for _o in p.outputs_to_be_spent { + // let mut new_output = NewOutputSql::new(o, OutputStatus::EncumberedToBeSpent, Some(p.tx_id))?; + // self.encrypt_if_necessary(&mut new_output)?; + // new_output.commit(&(*conn))?; + unimplemented!("Should not be any spent outputs") } for o in p.outputs_to_be_received { let mut new_output = NewOutputSql::new(o, OutputStatus::EncumberedToBeReceived, Some(p.tx_id))?; @@ -343,56 +378,48 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { } }, DbKeyValuePair::KeyManagerState(km) => { - let mut km_sql = KeyManagerStateSql::from(km); + let mut km_sql = NewKeyManagerStateSql::from(km); self.encrypt_if_necessary(&mut km_sql)?; - km_sql.set_state(&(*conn))? + km_sql.commit(&(*conn))? }, DbKeyValuePair::KnownOneSidedPaymentScripts(script) => { let mut script_sql = KnownOneSidedPaymentScriptSql::from(script); self.encrypt_if_necessary(&mut script_sql)?; script_sql.commit(&(*conn))? }, - DbKeyValuePair::UpdateOutputStatus(commitment, status) => { - let output = OutputSql::find_by_commitment(&commitment.to_vec(), &(*conn))?; - output.update( - UpdateOutput { - status: Some(status), - tx_id: None, - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, - }, - &(*conn), - )?; - }, }, WriteOperation::Remove(k) => match k { - DbKey::SpentOutput(s) => match OutputSql::find_status(&s.to_vec(), OutputStatus::Spent, &(*conn)) { - Ok(o) => { - o.delete(&(*conn))?; - return Ok(Some(DbValue::SpentOutput(Box::new(DbUnblindedOutput::try_from(o)?)))); - }, - Err(e) => { - match e { - OutputManagerStorageError::DieselError(DieselError::NotFound) => (), - e => return Err(e), - }; - }, + DbKey::SpentOutput(_s) => { + // match OutputSql::find_status(&s.to_vec(), OutputStatus::Spent, &(*conn)) { + unimplemented!("Deprecated") + // Ok(o) => { + // o.delete(&(*conn))?; + // return Ok(Some(DbValue::SpentOutput(Box::new(DbUnblindedOutput::try_from(o)?)))); + // }, + // Err(e) => { + // match e { + // OutputManagerStorageError::DieselError(DieselError::NotFound) => (), + // e => return Err(e), + // }; + // }, }, - DbKey::UnspentOutput(k) => match OutputSql::find_status(&k.to_vec(), OutputStatus::Unspent, &(*conn)) { - Ok(o) => { - o.delete(&(*conn))?; - return Ok(Some(DbValue::UnspentOutput(Box::new(DbUnblindedOutput::try_from(o)?)))); - }, - Err(e) => { - match e { - OutputManagerStorageError::DieselError(DieselError::NotFound) => (), - e => return Err(e), - }; - }, + DbKey::UnspentOutput(_k) => { + // match OutputSql::find_status(&k.to_vec(), OutputStatus::Unspent, &(*conn)) { + unimplemented!("Deprecated") + // Ok(o) => { + // o.delete(&(*conn))?; + // return Ok(Some(DbValue::UnspentOutput(Box::new(DbUnblindedOutput::try_from(o)?)))); + // }, + // Err(e) => { + // match e { + // OutputManagerStorageError::DieselError(DieselError::NotFound) => (), + // e => return Err(e), + // }; + // }, }, DbKey::AnyOutputByCommitment(commitment) => { + // Used by coinbase when mining. + match OutputSql::find_by_commitment(&commitment.to_vec(), &(*conn)) { Ok(o) => { o.delete(&(*conn))?; @@ -406,30 +433,32 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { }, } }, - DbKey::PendingTransactionOutputs(tx_id) => match PendingTransactionOutputSql::find(tx_id, &(*conn)) { - Ok(p) => { - let mut outputs = OutputSql::find_by_tx_id_and_encumbered(p.tx_id as u64, &(*conn))?; - - for o in outputs.iter_mut() { - self.decrypt_if_necessary(o)?; - } - - p.delete(&(*conn))?; - return Ok(Some(DbValue::PendingTransactionOutputs(Box::new( - pending_transaction_outputs_from_sql_outputs( - p.tx_id as u64, - &p.timestamp, - outputs, - p.coinbase_block_height.map(|h| h as u64), - )?, - )))); - }, - Err(e) => { - match e { - OutputManagerStorageError::DieselError(DieselError::NotFound) => (), - e => return Err(e), - }; - }, + DbKey::PendingTransactionOutputs(_tx_id) => { + // match PendingTransactionOutputSql::find(tx_id, &(*conn)) { + unimplemented!("Deprecated"); + // Ok(p) => { + // let mut outputs = OutputSql::find_by_tx_id_and_encumbered(p.tx_id as u64, &(*conn))?; + // + // for o in outputs.iter_mut() { + // self.decrypt_if_necessary(o)?; + // } + // + // p.delete(&(*conn))?; + // return Ok(Some(DbValue::PendingTransactionOutputs(Box::new( + // pending_transaction_outputs_from_sql_outputs( + // p.tx_id as u64, + // &p.timestamp, + // outputs, + // p.coinbase_block_height.map(|h| h as u64), + // )?, + // )))); + // }, + // Err(e) => { + // match e { + // OutputManagerStorageError::DieselError(DieselError::NotFound) => (), + // e => return Err(e), + // }; + // }, }, DbKey::UnspentOutputs => return Err(OutputManagerStorageError::OperationNotSupported), DbKey::SpentOutputs => return Err(OutputManagerStorageError::OperationNotSupported), @@ -445,7 +474,88 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { Ok(None) } - fn confirm_transaction(&self, tx_id: u64) -> Result<(), OutputManagerStorageError> { + fn set_output_to_unmined(&self, hash: Vec) -> Result<(), OutputManagerStorageError> { + let conn = self.database_connection.acquire_lock(); + // Only allow updating of non-deleted utxos + diesel::update(outputs::table.filter(outputs::hash.eq(hash).and(outputs::marked_deleted_at_height.is_null()))) + .set(( + outputs::mined_height.eq::>(None), + outputs::mined_in_block.eq::>>(None), + outputs::mined_mmr_position.eq::>(None), + outputs::status.eq(OutputStatus::Invalid as i32), + )) + .execute(&(*conn)) + .num_rows_affected_or_not_found(1)?; + + Ok(()) + } + + fn set_output_mined_height( + &self, + hash: Vec, + mined_height: u64, + mined_in_block: Vec, + mmr_position: u64, + ) -> Result<(), OutputManagerStorageError> { + let conn = self.database_connection.acquire_lock(); + // Only allow updating of non-deleted utxos + diesel::update(outputs::table.filter(outputs::hash.eq(hash).and(outputs::marked_deleted_at_height.is_null()))) + .set(( + outputs::mined_height.eq(mined_height as i64), + outputs::mined_in_block.eq(mined_in_block), + outputs::mined_mmr_position.eq(mmr_position as i64), + outputs::status.eq(OutputStatus::Unspent as i32), + )) + .execute(&(*conn)) + .num_rows_affected_or_not_found(1)?; + + Ok(()) + } + + fn mark_output_as_spent( + &self, + hash: Vec, + mark_deleted_at_height: u64, + mark_deleted_in_block: Vec, + ) -> Result<(), OutputManagerStorageError> { + let conn = self.database_connection.acquire_lock(); + // Only allow updating of non-deleted utxos + diesel::update(outputs::table.filter(outputs::hash.eq(hash).and(outputs::marked_deleted_at_height.is_null()))) + .set(( + outputs::marked_deleted_at_height.eq(mark_deleted_at_height as i64), + outputs::marked_deleted_in_block.eq(mark_deleted_in_block), + outputs::status.eq(OutputStatus::Spent as i32), + )) + .execute(&(*conn)) + .num_rows_affected_or_not_found(1)?; + + Ok(()) + } + + fn mark_output_as_unspent(&self, hash: Vec) -> Result<(), OutputManagerStorageError> { + let conn = self.database_connection.acquire_lock(); + // Only allow updating of non-deleted utxos + debug!(target: LOG_TARGET, "mark_output_as_unspent({})", hash.to_hex()); + diesel::update( + outputs::table.filter( + outputs::hash + .eq(hash) + .and(outputs::marked_deleted_at_height.is_not_null()) + .and(outputs::mined_height.is_not_null()), + ), + ) + .set(( + outputs::marked_deleted_at_height.eq::>(None), + outputs::marked_deleted_in_block.eq::>>(None), + outputs::status.eq(OutputStatus::Unspent as i32), + )) + .execute(&(*conn)) + .num_rows_affected_or_not_found(1)?; + + Ok(()) + } + + fn confirm_transaction_encumberance(&self, tx_id: u64) -> Result<(), OutputManagerStorageError> { let conn = self.database_connection.acquire_lock(); match PendingTransactionOutputSql::find(tx_id, &(*conn)) { @@ -457,11 +567,7 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { o.update( UpdateOutput { status: Some(OutputStatus::Unspent), - tx_id: None, - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, + ..Default::default() }, &(*conn), )?; @@ -469,11 +575,7 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { o.update( UpdateOutput { status: Some(OutputStatus::Spent), - tx_id: None, - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, + ..Default::default() }, &(*conn), )?; @@ -516,11 +618,8 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { o.update( UpdateOutput { status: Some(OutputStatus::EncumberedToBeSpent), - tx_id: Some(tx_id), - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, + spent_in_tx_id: Some(Some(tx_id)), + ..Default::default() }, &(*conn), )?; @@ -568,6 +667,32 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { Ok(()) } + fn get_last_mined_output(&self) -> Result, OutputManagerStorageError> { + let conn = self.database_connection.acquire_lock(); + + let output = OutputSql::first_by_mined_height_desc(&(*conn))?; + match output { + Some(mut o) => { + self.decrypt_if_necessary(&mut o)?; + Ok(Some(o.try_into()?)) + }, + None => Ok(None), + } + } + + fn get_last_spent_output(&self) -> Result, OutputManagerStorageError> { + let conn = self.database_connection.acquire_lock(); + + let output = OutputSql::first_by_marked_deleted_height_desc(&(*conn))?; + match output { + Some(mut o) => { + self.decrypt_if_necessary(&mut o)?; + Ok(Some(o.try_into()?)) + }, + None => Ok(None), + } + } + fn cancel_pending_transaction(&self, tx_id: u64) -> Result<(), OutputManagerStorageError> { let conn = self.database_connection.acquire_lock(); @@ -580,11 +705,7 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { o.update( UpdateOutput { status: Some(OutputStatus::CancelledInbound), - tx_id: None, - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, + ..Default::default() }, &(*conn), )?; @@ -592,11 +713,8 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { o.update( UpdateOutput { status: Some(OutputStatus::Unspent), - tx_id: None, - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, + spent_in_tx_id: Some(None), + ..Default::default() }, &(*conn), )?; @@ -656,11 +774,7 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { output.update( UpdateOutput { status: Some(OutputStatus::Invalid), - tx_id: None, - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, + ..Default::default() }, &(*conn), )?; @@ -673,12 +787,9 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { let db_output = OutputSql::find_by_commitment_and_cancelled(&output.commitment.to_vec(), false, &conn)?; db_output.update( UpdateOutput { - status: None, - tx_id: None, - spending_key: None, - script_private_key: None, metadata_signature_nonce: Some(output.metadata_signature.public_nonce().to_vec()), metadata_signature_u_key: Some(output.metadata_signature.u().to_vec()), + ..Default::default() }, &(*conn), )?; @@ -696,11 +807,7 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { output.update( UpdateOutput { status: Some(OutputStatus::Unspent), - tx_id: None, - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, + ..Default::default() }, &(*conn), )?; @@ -721,11 +828,8 @@ impl OutputManagerBackend for OutputManagerSqliteDatabase { let mut o = output.update( UpdateOutput { status: Some(OutputStatus::Unspent), - tx_id: None, - spending_key: None, - script_private_key: None, - metadata_signature_nonce: None, - metadata_signature_u_key: None, + spent_in_tx_id: Some(None), + ..Default::default() }, &(*conn), )?; @@ -897,6 +1001,7 @@ struct NewOutputSql { flags: i32, maturity: i64, status: i32, + // Deprecated tx_id: Option, hash: Option>, script: Vec, @@ -906,13 +1011,14 @@ struct NewOutputSql { metadata_signature_nonce: Vec, metadata_signature_u_key: Vec, metadata_signature_v_key: Vec, + received_in_tx_id: Option, } impl NewOutputSql { pub fn new( output: DbUnblindedOutput, status: OutputStatus, - tx_id: Option, + received_in_tx_id: Option, ) -> Result { Ok(Self { commitment: Some(output.commitment.to_vec()), @@ -921,7 +1027,8 @@ impl NewOutputSql { flags: output.unblinded_output.features.flags.bits() as i32, maturity: output.unblinded_output.features.maturity as i64, status: status as i32, - tx_id: tx_id.map(|i| i as i64), + tx_id: None, + received_in_tx_id: received_in_tx_id.map(|i| i as i64), hash: Some(output.hash), script: output.unblinded_output.script.as_bytes(), input_data: output.unblinded_output.input_data.as_bytes(), @@ -957,13 +1064,14 @@ impl Encryptable for NewOutputSql { #[derive(Clone, Debug, Queryable, Identifiable, PartialEq)] #[table_name = "outputs"] struct OutputSql { - id: i32, + id: i32, // Auto inc primary key commitment: Option>, spending_key: Vec, value: i64, flags: i32, maturity: i64, status: i32, + #[deprecated] tx_id: Option, hash: Option>, script: Vec, @@ -973,6 +1081,13 @@ struct OutputSql { metadata_signature_nonce: Vec, metadata_signature_u_key: Vec, metadata_signature_v_key: Vec, + mined_height: Option, + mined_in_block: Option>, + mined_mmr_position: Option, + marked_deleted_at_height: Option, + marked_deleted_in_block: Option>, + received_in_tx_id: Option, + spent_in_tx_id: Option, } impl OutputSql { @@ -997,6 +1112,42 @@ impl OutputSql { .load(conn)?) } + pub fn index_mined_in_block_is_null(conn: &SqliteConnection) -> Result, OutputManagerStorageError> { + Ok(outputs::table + .filter(outputs::mined_in_block.is_null()) + .order(outputs::id.asc()) + .load(conn)?) + } + + pub fn index_marked_deleted_in_block_is_null( + conn: &SqliteConnection, + ) -> Result, OutputManagerStorageError> { + Ok(outputs::table + .filter(outputs::marked_deleted_in_block.is_null()) + // Only return mined + .filter(outputs::mined_in_block.is_not_null()) + .order(outputs::id.asc()) + .load(conn)?) + } + + pub fn first_by_mined_height_desc(conn: &SqliteConnection) -> Result, OutputManagerStorageError> { + Ok(outputs::table + .filter(outputs::mined_height.is_not_null()) + .order(outputs::mined_height.desc()) + .first(conn) + .optional()?) + } + + pub fn first_by_marked_deleted_height_desc( + conn: &SqliteConnection, + ) -> Result, OutputManagerStorageError> { + Ok(outputs::table + .filter(outputs::marked_deleted_at_height.is_not_null()) + .order(outputs::marked_deleted_at_height.desc()) + .first(conn) + .optional()?) + } + /// Find a particular Output, if it exists pub fn find(spending_key: &[u8], conn: &SqliteConnection) -> Result { Ok(outputs::table @@ -1084,15 +1235,10 @@ impl OutputSql { updated_output: UpdateOutput, conn: &SqliteConnection, ) -> Result { - let num_updated = diesel::update(outputs::table.filter(outputs::id.eq(&self.id))) + diesel::update(outputs::table.filter(outputs::id.eq(&self.id))) .set(UpdateOutputSql::from(updated_output)) - .execute(conn)?; - - if num_updated == 0 { - return Err(OutputManagerStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + .execute(conn) + .num_rows_affected_or_not_found(1)?; OutputSql::find(&self.spending_key, conn) } @@ -1103,15 +1249,10 @@ impl OutputSql { updated_null: NullOutputSql, conn: &SqliteConnection, ) -> Result { - let num_updated = diesel::update(outputs::table.filter(outputs::spending_key.eq(&self.spending_key))) + diesel::update(outputs::table.filter(outputs::spending_key.eq(&self.spending_key))) .set(updated_null) - .execute(conn)?; - - if num_updated == 0 { - return Err(OutputManagerStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + .execute(conn) + .num_rows_affected_or_not_found(1)?; OutputSql::find(&self.spending_key, conn) } @@ -1120,12 +1261,9 @@ impl OutputSql { pub fn update_encryption(&self, conn: &SqliteConnection) -> Result<(), OutputManagerStorageError> { let _ = self.update( UpdateOutput { - status: None, - tx_id: None, spending_key: Some(self.spending_key.clone()), script_private_key: Some(self.script_private_key.clone()), - metadata_signature_nonce: None, - metadata_signature_u_key: None, + ..Default::default() }, conn, )?; @@ -1213,6 +1351,11 @@ impl TryFrom for DbUnblindedOutput { commitment, unblinded_output, hash, + mined_height: o.mined_height.map(|mh| mh as u64), + mined_in_block: o.mined_in_block, + mined_mmr_position: o.mined_mmr_position.map(|mp| mp as u64), + marked_deleted_at_height: o.marked_deleted_at_height.map(|d| d as u64), + marked_deleted_in_block: o.marked_deleted_in_block, }) } } @@ -1249,6 +1392,7 @@ impl From for NewOutputSql { metadata_signature_nonce: o.metadata_signature_nonce, metadata_signature_u_key: o.metadata_signature_u_key, metadata_signature_v_key: o.metadata_signature_v_key, + received_in_tx_id: o.received_in_tx_id, } } } @@ -1260,9 +1404,11 @@ impl PartialEq for OutputSql { } /// These are the fields that can be updated for an Output +#[derive(Default)] pub struct UpdateOutput { status: Option, - tx_id: Option, + received_in_tx_id: Option>, + spent_in_tx_id: Option>, spending_key: Option>, script_private_key: Option>, metadata_signature_nonce: Option>, @@ -1273,7 +1419,8 @@ pub struct UpdateOutput { #[table_name = "outputs"] pub struct UpdateOutputSql { status: Option, - tx_id: Option, + received_in_tx_id: Option>, + spent_in_tx_id: Option>, spending_key: Option>, script_private_key: Option>, metadata_signature_nonce: Option>, @@ -1293,11 +1440,12 @@ impl From for UpdateOutputSql { fn from(u: UpdateOutput) -> Self { Self { status: u.status.map(|t| t as i32), - tx_id: u.tx_id.map(|t| t as i64), spending_key: u.spending_key, script_private_key: u.script_private_key, metadata_signature_nonce: u.metadata_signature_nonce, metadata_signature_u_key: u.metadata_signature_u_key, + received_in_tx_id: u.received_in_tx_id.map(|o| o.map(|t| t as i64)), + spent_in_tx_id: u.spent_in_tx_id.map(|o| o.map(|t| t as i64)), } } } @@ -1392,17 +1540,10 @@ impl PendingTransactionOutputSql { &self, conn: &SqliteConnection, ) -> Result { - let num_updated = diesel::update( - pending_transaction_outputs::table.filter(pending_transaction_outputs::tx_id.eq(&self.tx_id)), - ) - .set(UpdatePendingTransactionOutputSql { short_term: Some(0i32) }) - .execute(conn)?; - - if num_updated == 0 { - return Err(OutputManagerStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + diesel::update(pending_transaction_outputs::table.filter(pending_transaction_outputs::tx_id.eq(&self.tx_id))) + .set(UpdatePendingTransactionOutputSql { short_term: Some(0i32) }) + .execute(conn) + .num_rows_affected_or_not_found(1)?; PendingTransactionOutputSql::find(self.tx_id as u64, conn) } @@ -1414,20 +1555,28 @@ pub struct UpdatePendingTransactionOutputSql { short_term: Option, } -#[derive(Clone, Debug, Queryable, Insertable)] +#[derive(Clone, Debug, Queryable, Identifiable)] #[table_name = "key_manager_states"] struct KeyManagerStateSql { - id: Option, + id: i32, master_key: Vec, branch_seed: String, primary_key_index: i64, timestamp: NaiveDateTime, } -impl From for KeyManagerStateSql { +#[derive(Clone, Debug, Insertable)] +#[table_name = "key_manager_states"] +struct NewKeyManagerStateSql { + master_key: Vec, + branch_seed: String, + primary_key_index: i64, + timestamp: NaiveDateTime, +} + +impl From for NewKeyManagerStateSql { fn from(km: KeyManagerState) -> Self { Self { - id: None, master_key: km.master_key.to_vec(), branch_seed: km.branch_seed, primary_key_index: km.primary_key_index as i64, @@ -1435,7 +1584,6 @@ impl From for KeyManagerStateSql { } } } - impl TryFrom for KeyManagerState { type Error = OutputManagerStorageError; @@ -1448,14 +1596,16 @@ impl TryFrom for KeyManagerState { } } -impl KeyManagerStateSql { +impl NewKeyManagerStateSql { fn commit(&self, conn: &SqliteConnection) -> Result<(), OutputManagerStorageError> { diesel::insert_into(key_manager_states::table) .values(self.clone()) .execute(conn)?; Ok(()) } +} +impl KeyManagerStateSql { pub fn get_state(conn: &SqliteConnection) -> Result { key_manager_states::table .first::(conn) @@ -1471,16 +1621,20 @@ impl KeyManagerStateSql { primary_key_index: Some(self.primary_key_index), }; - let num_updated = diesel::update(key_manager_states::table.filter(key_manager_states::id.eq(&km.id))) + diesel::update(key_manager_states::table.filter(key_manager_states::id.eq(&km.id))) .set(update) - .execute(conn)?; - if num_updated == 0 { - return Err(OutputManagerStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + .execute(conn) + .num_rows_affected_or_not_found(1)?; + }, + Err(_) => { + let inserter = NewKeyManagerStateSql { + master_key: self.master_key.clone(), + branch_seed: self.branch_seed.clone(), + primary_key_index: self.primary_key_index, + timestamp: self.timestamp, + }; + inserter.commit(conn)?; }, - Err(_) => self.commit(conn)?, } Ok(()) } @@ -1494,14 +1648,10 @@ impl KeyManagerStateSql { branch_seed: None, primary_key_index: Some(current_index), }; - let num_updated = diesel::update(key_manager_states::table.filter(key_manager_states::id.eq(&km.id))) + diesel::update(key_manager_states::table.filter(key_manager_states::id.eq(&km.id))) .set(update) - .execute(conn)?; - if num_updated == 0 { - return Err(OutputManagerStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + .execute(conn) + .num_rows_affected_or_not_found(1)?; current_index }, Err(_) => return Err(OutputManagerStorageError::KeyManagerNotInitialized), @@ -1516,14 +1666,10 @@ impl KeyManagerStateSql { branch_seed: None, primary_key_index: Some(index as i64), }; - let num_updated = diesel::update(key_manager_states::table.filter(key_manager_states::id.eq(&km.id))) + diesel::update(key_manager_states::table.filter(key_manager_states::id.eq(&km.id))) .set(update) - .execute(conn)?; - if num_updated == 0 { - return Err(OutputManagerStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + .execute(conn) + .num_rows_affected_or_not_found(1)?; Ok(()) }, Err(_) => Err(OutputManagerStorageError::KeyManagerNotInitialized), @@ -1561,6 +1707,29 @@ impl Encryptable for KeyManagerStateSql { } } +impl Encryptable for NewKeyManagerStateSql { + fn encrypt(&mut self, cipher: &Aes256Gcm) -> Result<(), Error> { + let encrypted_master_key = encrypt_bytes_integral_nonce(&cipher, self.master_key.clone())?; + let encrypted_branch_seed = + encrypt_bytes_integral_nonce(&cipher, self.branch_seed.clone().as_bytes().to_vec())?; + self.master_key = encrypted_master_key; + self.branch_seed = encrypted_branch_seed.to_hex(); + Ok(()) + } + + fn decrypt(&mut self, _cipher: &Aes256Gcm) -> Result<(), Error> { + unimplemented!("Not supported") + // let decrypted_master_key = decrypt_bytes_integral_nonce(&cipher, self.master_key.clone())?; + // let decrypted_branch_seed = + // decrypt_bytes_integral_nonce(&cipher, from_hex(self.branch_seed.as_str()).map_err(|_| Error)?)?; + // self.master_key = decrypted_master_key; + // self.branch_seed = from_utf8(decrypted_branch_seed.as_slice()) + // .map_err(|_| Error)? + // .to_string(); + // Ok(()) + } +} + #[derive(Clone, Debug, Queryable, Insertable, Identifiable, PartialEq, AsChangeset)] #[table_name = "known_one_sided_payment_scripts"] #[primary_key(script_hash)] @@ -1624,18 +1793,13 @@ impl KnownOneSidedPaymentScriptSql { updated_known_script: UpdateKnownOneSidedPaymentScript, conn: &SqliteConnection, ) -> Result { - let num_updated = diesel::update( + diesel::update( known_one_sided_payment_scripts::table .filter(known_one_sided_payment_scripts::script_hash.eq(&self.script_hash)), ) .set(updated_known_script) - .execute(conn)?; - - if num_updated == 0 { - return Err(OutputManagerStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + .execute(conn) + .num_rows_affected_or_not_found(1)?; KnownOneSidedPaymentScriptSql::find(&self.script_hash, conn) } diff --git a/base_layer/wallet/src/output_manager_service/tasks/mod.rs b/base_layer/wallet/src/output_manager_service/tasks/mod.rs index 0c28ca2c90..6ed7a3f116 100644 --- a/base_layer/wallet/src/output_manager_service/tasks/mod.rs +++ b/base_layer/wallet/src/output_manager_service/tasks/mod.rs @@ -20,6 +20,24 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -mod txo_validation_task; +mod txo_validation_task_v2; -pub use txo_validation_task::{TxoValidationTask, TxoValidationType}; +use std::fmt; +pub use txo_validation_task_v2::TxoValidationTaskV2; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TxoValidationType { + Unspent, + Spent, + Invalid, +} + +impl fmt::Display for TxoValidationType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TxoValidationType::Unspent => write!(f, "Unspent Outputs Validation"), + TxoValidationType::Spent => write!(f, "Spent Outputs Validation"), + TxoValidationType::Invalid => write!(f, "Invalid Outputs Validation"), + } + } +} diff --git a/base_layer/wallet/src/output_manager_service/tasks/txo_validation_task.rs b/base_layer/wallet/src/output_manager_service/tasks/txo_validation_task.rs index e3e022e4fb..19919b0729 100644 --- a/base_layer/wallet/src/output_manager_service/tasks/txo_validation_task.rs +++ b/base_layer/wallet/src/output_manager_service/tasks/txo_validation_task.rs @@ -636,20 +636,3 @@ where TBackend: OutputManagerBackend + 'static self.retry_delay = new_delay; } } - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum TxoValidationType { - Unspent, - Spent, - Invalid, -} - -impl fmt::Display for TxoValidationType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - TxoValidationType::Unspent => write!(f, "Unspent Outputs Validation"), - TxoValidationType::Spent => write!(f, "Spent Outputs Validation"), - TxoValidationType::Invalid => write!(f, "Invalid Outputs Validation"), - } - } -} diff --git a/base_layer/wallet/src/output_manager_service/tasks/txo_validation_task_v2.rs b/base_layer/wallet/src/output_manager_service/tasks/txo_validation_task_v2.rs new file mode 100644 index 0000000000..a5794f90d9 --- /dev/null +++ b/base_layer/wallet/src/output_manager_service/tasks/txo_validation_task_v2.rs @@ -0,0 +1,408 @@ +// Copyright 2021. The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::output_manager_service::{ + config::OutputManagerServiceConfig, + error::{OutputManagerError, OutputManagerProtocolError, OutputManagerProtocolErrorExt}, + handle::{OutputManagerEvent, OutputManagerEventSender}, + storage::{ + database::{OutputManagerBackend, OutputManagerDatabase}, + models::DbUnblindedOutput, + }, + TxoValidationType, +}; +use log::*; +use std::{collections::HashMap, convert::TryInto, sync::Arc}; +use tari_common_types::types::BlockHash; +use tari_comms::{ + connectivity::ConnectivityRequester, + protocol::rpc::{RpcError::RequestFailed, RpcStatusCode::NotFound}, + types::CommsPublicKey, +}; +use tari_core::{ + base_node::{rpc::BaseNodeWalletRpcClient, sync::rpc::BaseNodeSyncRpcClient}, + blocks::BlockHeader, + proto::base_node::{QueryDeletedRequest, UtxoQueryRequest}, +}; +use tari_crypto::tari_utilities::{hex::Hex, Hashable}; +use tari_shutdown::ShutdownSignal; + +const LOG_TARGET: &str = "wallet::output_service::txo_validation_task_v2"; + +pub struct TxoValidationTaskV2 { + base_node_pk: CommsPublicKey, + operation_id: u64, + batch_size: usize, + db: OutputManagerDatabase, + connectivity_requester: ConnectivityRequester, + event_publisher: OutputManagerEventSender, + validation_type: TxoValidationType, + config: OutputManagerServiceConfig, +} + +impl TxoValidationTaskV2 +where TBackend: OutputManagerBackend + 'static +{ + pub fn new( + base_node_pk: CommsPublicKey, + operation_id: u64, + batch_size: usize, + db: OutputManagerDatabase, + connectivity_requester: ConnectivityRequester, + event_publisher: OutputManagerEventSender, + validation_type: TxoValidationType, + config: OutputManagerServiceConfig, + ) -> Self { + Self { + base_node_pk, + operation_id, + batch_size, + db, + connectivity_requester, + event_publisher, + validation_type, + config, + } + } + + pub async fn execute(mut self, _shutdown: ShutdownSignal) -> Result { + let (mut sync_client, mut wallet_client) = self.create_base_node_clients().await?; + + info!( + target: LOG_TARGET, + "Starting TXO validation protocol V2 (Id: {}) for {}", self.operation_id, self.validation_type, + ); + + let last_mined_header = self.check_for_reorgs(&mut sync_client).await?; + + self.update_received_outputs(&mut wallet_client).await?; + + self.update_spent_outputs(&mut wallet_client, last_mined_header).await?; + self.publish_event(OutputManagerEvent::TxoValidationSuccess( + self.operation_id, + self.validation_type, + )); + Ok(self.operation_id) + } + + async fn update_spent_outputs( + &self, + wallet_client: &mut BaseNodeWalletRpcClient, + last_mined_header_hash: Option, + ) -> Result<(), OutputManagerProtocolError> { + let unmined_outputs = self + .db + .fetch_unmined_spent_outputs() + .await + .for_protocol(self.operation_id) + .unwrap(); + + if unmined_outputs.is_empty() { + return Ok(()); + } + + for batch in unmined_outputs.chunks(self.batch_size) { + info!( + target: LOG_TARGET, + "Asking base node for status of {} mmr_positions", + batch.len() + ); + + // We have to send positions to the base node because if the base node cannot find the hash of the output + // we can't tell if the output ever existed, as opposed to existing and was spent. + // This assumes that the base node has not reorged since the last time we asked. + let deleted_bitmap_response = wallet_client + .query_deleted(QueryDeletedRequest { + chain_must_include_header: last_mined_header_hash.clone(), + mmr_positions: batch.iter().filter_map(|ub| ub.mined_mmr_position).collect(), + }) + .await + .for_protocol(self.operation_id)?; + + for output in batch { + if deleted_bitmap_response + .deleted_positions + .contains(&output.mined_mmr_position.unwrap()) + { + self.db + .mark_output_as_spent( + output.hash.clone(), + deleted_bitmap_response.height_of_longest_chain, + deleted_bitmap_response.best_block.clone(), + ) + .await + .for_protocol(self.operation_id)?; + } + if deleted_bitmap_response + .not_deleted_positions + .contains(&output.mined_mmr_position.unwrap()) && + output.marked_deleted_at_height.is_some() + { + self.db + .mark_output_as_unspent(output.hash.clone()) + .await + .for_protocol(self.operation_id)?; + } + } + } + Ok(()) + } + + async fn update_received_outputs( + &self, + wallet_client: &mut BaseNodeWalletRpcClient, + ) -> Result<(), OutputManagerProtocolError> { + let unmined_outputs = self + .db + .fetch_unmined_received_outputs() + .await + .for_protocol(self.operation_id) + .unwrap(); + + for batch in unmined_outputs.chunks(self.batch_size) { + info!( + target: LOG_TARGET, + "Asking base node for location of {} mined outputs by hash", + batch.len() + ); + let (mined, unmined) = self + .query_base_node_for_outputs(batch, wallet_client) + .await + .for_protocol(self.operation_id)?; + info!( + target: LOG_TARGET, + "Base node returned {} as mined and {} as unmined", + mined.len(), + unmined.len() + ); + for (tx, mined_height, mined_in_block, mmr_position) in &mined { + info!( + target: LOG_TARGET, + "Updating output comm:{}: hash{} as mined", + tx.commitment.to_hex(), + tx.hash.to_hex() + ); + self.update_output_as_mined(&tx, mined_in_block, *mined_height, *mmr_position) + .await?; + } + } + + Ok(()) + } + + async fn create_base_node_clients( + &mut self, + ) -> Result<(BaseNodeSyncRpcClient, BaseNodeWalletRpcClient), OutputManagerProtocolError> { + let mut base_node_connection = self + .connectivity_requester + .dial_peer(self.base_node_pk.clone().into()) + .await + .for_protocol(self.operation_id)?; + let sync_client = base_node_connection + .connect_rpc_using_builder( + BaseNodeSyncRpcClient::builder().with_deadline(self.config.base_node_query_timeout), + ) + .await + .for_protocol(self.operation_id)?; + let wallet_client = base_node_connection + .connect_rpc_using_builder( + BaseNodeWalletRpcClient::builder().with_deadline(self.config.base_node_query_timeout), + ) + .await + .for_protocol(self.operation_id)?; + + Ok((sync_client, wallet_client)) + } + + // returns the last header found still in the chain + async fn check_for_reorgs( + &mut self, + client: &mut BaseNodeSyncRpcClient, + ) -> Result, OutputManagerProtocolError> { + let mut last_mined_header_hash = None; + info!( + target: LOG_TARGET, + "Checking last mined TXO to see if the base node has re-orged" + ); + + while let Some(last_spent_output) = self + .db + .get_last_spent_output() + .await + .for_protocol(self.operation_id) + .unwrap() + { + let mined_height = last_spent_output.marked_deleted_at_height.unwrap(); // TODO: fix unwrap + let mined_in_block_hash = last_spent_output.marked_deleted_in_block.clone().unwrap(); // TODO: fix unwrap. + let block_at_height = self + .get_base_node_block_at_height(mined_height, client) + .await + .for_protocol(self.operation_id)?; + if block_at_height.is_none() || block_at_height.unwrap() != mined_in_block_hash { + // Chain has reorged since we last + warn!( + target: LOG_TARGET, + "The block that output (commitment) was spent in has been reorged out, will try to find this \ + output again, but these funds have potentially been re-orged out of the chain", + ); + self.db + .mark_output_as_unspent(last_spent_output.hash.clone()) + .await + .for_protocol(self.operation_id)?; + } else { + info!( + target: LOG_TARGET, + "Last mined transaction is still in the block chain according to base node." + ); + break; + } + } + + while let Some(last_mined_output) = self + .db + .get_last_mined_output() + .await + .for_protocol(self.operation_id) + .unwrap() + { + let mined_height = last_mined_output.mined_height.unwrap(); // TODO: fix unwrap + let mined_in_block_hash = last_mined_output.mined_in_block.clone().unwrap(); // TODO: fix unwrap. + let block_at_height = self + .get_base_node_block_at_height(mined_height, client) + .await + .for_protocol(self.operation_id)?; + if block_at_height.is_none() || block_at_height.unwrap() != mined_in_block_hash { + // Chain has reorged since we last + warn!( + target: LOG_TARGET, + "The block that output (commitment) was in has been reorged out, will try to find this output \ + again, but these funds have potentially been re-orged out of the chain", + ); + self.db + .set_output_as_unmined(last_mined_output.hash.clone()) + .await + .for_protocol(self.operation_id)?; + } else { + info!( + target: LOG_TARGET, + "Last mined transaction is still in the block chain according to base node." + ); + last_mined_header_hash = Some(mined_in_block_hash); + break; + } + } + Ok(last_mined_header_hash) + } + + // TODO: remove this duplicated code from transaction validation protocol + + async fn get_base_node_block_at_height( + &mut self, + height: u64, + client: &mut BaseNodeSyncRpcClient, + ) -> Result, OutputManagerError> { + let result = match client.get_header_by_height(height).await { + Ok(r) => r, + Err(rpc_error) => { + warn!(target: LOG_TARGET, "Error asking base node for header:{}", rpc_error); + match &rpc_error { + RequestFailed(status) => { + if status.status_code() == NotFound { + return Ok(None); + } else { + return Err(rpc_error.into()); + } + }, + _ => { + return Err(rpc_error.into()); + }, + } + }, + }; + + let block_header: BlockHeader = result + .try_into() + .map_err(|s| OutputManagerError::InvalidMessageError(format!("Could not convert block header: {}", s)))?; + Ok(Some(block_header.hash())) + } + + async fn query_base_node_for_outputs( + &self, + batch: &[DbUnblindedOutput], + base_node_client: &mut BaseNodeWalletRpcClient, + ) -> Result<(Vec<(DbUnblindedOutput, u64, BlockHash, u64)>, Vec), OutputManagerError> { + let batch_hashes = batch.iter().map(|o| o.hash.clone()).collect(); + + let batch_response = base_node_client + .utxo_query(UtxoQueryRequest { + output_hashes: batch_hashes, + }) + .await?; + + let mut mined = vec![]; + let mut unmined = vec![]; + + let mut returned_outputs = HashMap::new(); + for output_proto in batch_response.responses.iter() { + returned_outputs.insert(output_proto.output_hash.clone(), output_proto); + } + + for output in batch { + if let Some(returned_output) = returned_outputs.get(&output.hash) { + mined.push(( + output.clone(), + returned_output.mined_height, + returned_output.mined_in_block.clone(), + returned_output.mmr_position, + )) + } else { + unmined.push(output.clone()); + } + } + + Ok((mined, unmined)) + } + + #[allow(clippy::ptr_arg)] + async fn update_output_as_mined( + &self, + tx: &DbUnblindedOutput, + mined_in_block: &BlockHash, + mined_height: u64, + mmr_position: u64, + ) -> Result<(), OutputManagerProtocolError> { + self.db + .set_output_mined_height(tx.hash.clone(), mined_height, mined_in_block.clone(), mmr_position) + .await + .for_protocol(self.operation_id)?; + + Ok(()) + } + + fn publish_event(&self, event: OutputManagerEvent) { + if let Err(e) = self.event_publisher.send(Arc::new(event)) { + debug!( + target: LOG_TARGET, + "Error sending event because there are no subscribers: {:?}", e + ); + } + } +} diff --git a/base_layer/wallet/src/schema.rs b/base_layer/wallet/src/schema.rs index 6b6c61512a..8658b2bba4 100644 --- a/base_layer/wallet/src/schema.rs +++ b/base_layer/wallet/src/schema.rs @@ -24,6 +24,7 @@ table! { valid -> Integer, confirmations -> Nullable, mined_height -> Nullable, + mined_in_block -> Nullable, } } @@ -51,7 +52,7 @@ table! { table! { key_manager_states (id) { - id -> Nullable, + id -> Integer, master_key -> Binary, branch_seed -> Text, primary_key_index -> BigInt, @@ -102,6 +103,13 @@ table! { metadata_signature_nonce -> Binary, metadata_signature_u_key -> Binary, metadata_signature_v_key -> Binary, + mined_height -> Nullable, + mined_in_block -> Nullable, + mined_mmr_position -> Nullable, + marked_deleted_at_height -> Nullable, + marked_deleted_in_block -> Nullable, + received_in_tx_id -> Nullable, + spent_in_tx_id -> Nullable, } } diff --git a/base_layer/wallet/src/transaction_service/error.rs b/base_layer/wallet/src/transaction_service/error.rs index c197dd2024..8d93b73159 100644 --- a/base_layer/wallet/src/transaction_service/error.rs +++ b/base_layer/wallet/src/transaction_service/error.rs @@ -27,7 +27,7 @@ use crate::{ use diesel::result::Error as DieselError; use futures::channel::oneshot::Canceled; use serde_json::Error as SerdeJsonError; -use tari_comms::{peer_manager::node_id::NodeIdError, protocol::rpc::RpcError}; +use tari_comms::{connectivity::ConnectivityError, peer_manager::node_id::NodeIdError, protocol::rpc::RpcError}; use tari_comms_dht::outbound::DhtOutboundError; use tari_core::transactions::{transaction::TransactionError, transaction_protocol::TransactionProtocolError}; use tari_p2p::services::liveness::error::LivenessError; @@ -78,6 +78,8 @@ pub enum TransactionServiceError { DiscoveryProcessFailed(TxId), #[error("Invalid Completed Transaction provided")] InvalidCompletedTransaction, + #[error("Attempted to broadcast a coinbase transaction. TxId `{0}`")] + AttemptedToBroadcastCoinbaseTransaction(TxId), #[error("No Base Node public keys are provided for Base chain broadcast and monitoring")] NoBaseNodeKeysProvided, #[error("Error sending data to Protocol via registered channels")] @@ -140,6 +142,12 @@ pub enum TransactionServiceError { ByteArrayError(#[from] tari_crypto::tari_utilities::ByteArrayError), #[error("Transaction Service Error: `{0}`")] ServiceError(String), + + #[error("Connectivity error: {source}")] + ConnectivityError { + #[from] + source: ConnectivityError, + }, } #[derive(Debug, Error)] @@ -199,3 +207,16 @@ impl From for TransactionServiceError { tspe.error } } + +pub trait TransactionServiceProtocolErrorExt { + fn for_protocol(self, id: u64) -> Result; +} + +impl> TransactionServiceProtocolErrorExt for Result { + fn for_protocol(self, id: u64) -> Result { + match self { + Ok(r) => Ok(r), + Err(e) => Err(TransactionServiceProtocolError::new(id, e.into())), + } + } +} diff --git a/base_layer/wallet/src/transaction_service/handle.rs b/base_layer/wallet/src/transaction_service/handle.rs index f34a5f667f..a2e3f6d4b4 100644 --- a/base_layer/wallet/src/transaction_service/handle.rs +++ b/base_layer/wallet/src/transaction_service/handle.rs @@ -64,7 +64,6 @@ pub enum TransactionServiceRequest { RestartBroadcastProtocols, GetNumConfirmationsRequired, SetNumConfirmationsRequired(u64), - SetCompletedTransactionValidity(u64, bool), ValidateTransactions(ValidationRetryStrategy), } @@ -107,10 +106,10 @@ impl fmt::Display for TransactionServiceRequest { Self::SetNumConfirmationsRequired(_) => f.write_str("SetNumConfirmationsRequired"), Self::GetAnyTransaction(t) => f.write_str(&format!("GetAnyTransaction({})", t)), TransactionServiceRequest::ValidateTransactions(t) => f.write_str(&format!("ValidateTransaction({:?})", t)), - TransactionServiceRequest::SetCompletedTransactionValidity(tx_id, s) => f.write_str(&format!( - "SetCompletedTransactionValidity(TxId: {}, Validity: {:?})", - tx_id, s - )), + /* TransactionServiceRequest::SetCompletedTransactionValidity(tx_id, s) => f.write_str(&format!( + * "SetCompletedTransactionValidity(TxId: {}, Validity: {:?})", + * tx_id, s + * )), */ } } } @@ -154,9 +153,17 @@ pub enum TransactionEvent { TransactionCancelled(TxId), TransactionBroadcast(TxId), TransactionImported(TxId), - TransactionMined(TxId), + TransactionMined { + tx_id: TxId, + is_valid: bool, + }, TransactionMinedRequestTimedOut(TxId), - TransactionMinedUnconfirmed(TxId, u64), + // TODO: Split into normal transaction mined and coinbase transaction mined + TransactionMinedUnconfirmed { + tx_id: TxId, + num_confirmations: u64, + is_valid: bool, + }, TransactionValidationTimedOut(u64), TransactionValidationSuccess(u64), TransactionValidationFailure(u64), @@ -521,15 +528,4 @@ impl TransactionServiceHandle { _ => Err(TransactionServiceError::UnexpectedApiResponse), } } - - pub async fn set_transaction_validity(&mut self, tx_id: TxId, valid: bool) -> Result<(), TransactionServiceError> { - match self - .handle - .call(TransactionServiceRequest::SetCompletedTransactionValidity(tx_id, valid)) - .await?? - { - TransactionServiceResponse::CompletedTransactionValidityChanged => Ok(()), - _ => Err(TransactionServiceError::UnexpectedApiResponse), - } - } } diff --git a/base_layer/wallet/src/transaction_service/protocols/mod.rs b/base_layer/wallet/src/transaction_service/protocols/mod.rs index 772c3402bd..4db0429a29 100644 --- a/base_layer/wallet/src/transaction_service/protocols/mod.rs +++ b/base_layer/wallet/src/transaction_service/protocols/mod.rs @@ -24,4 +24,4 @@ pub mod transaction_broadcast_protocol; pub mod transaction_coinbase_monitoring_protocol; pub mod transaction_receive_protocol; pub mod transaction_send_protocol; -pub mod transaction_validation_protocol; +pub mod transaction_validation_protocol_v2; diff --git a/base_layer/wallet/src/transaction_service/protocols/transaction_broadcast_protocol.rs b/base_layer/wallet/src/transaction_service/protocols/transaction_broadcast_protocol.rs index 05191c8f8d..1f17c906c2 100644 --- a/base_layer/wallet/src/transaction_service/protocols/transaction_broadcast_protocol.rs +++ b/base_layer/wallet/src/transaction_service/protocols/transaction_broadcast_protocol.rs @@ -276,35 +276,35 @@ where TBackend: TransactionBackend + 'static }, TxBroadcastMode::TransactionQuery => { if result? { - // We are done! - self.resources - .output_manager_service - .confirm_transaction( - completed_tx.tx_id, - completed_tx.transaction.body.inputs().clone(), - completed_tx.transaction.body.outputs().clone(), - ) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - - self.resources - .db - .confirm_broadcast_or_coinbase_transaction(completed_tx.tx_id) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionMined(self.tx_id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); + debug!(target: LOG_TARGET, "Transaction already mined, transaction validation protocol will continue from here "); + // self.resources + // .output_manager_service + // .confirm_transaction( + // completed_tx.tx_id, + // completed_tx.transaction.body.inputs().clone(), + // completed_tx.transaction.body.outputs().clone(), + // ) + // .await + // .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; + // + // self.resources + // .db + // .confirm_broadcast_or_coinbase_transaction(completed_tx.tx_id) + // .await + // .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; + // + // let _ = self + // .resources + // .event_publisher + // .send(Arc::new(TransactionEvent::TransactionMined(self.tx_id))) + // .map_err(|e| { + // trace!( + // target: LOG_TARGET, + // "Error sending event because there are no subscribers: {:?}", + // e + // ); + // e + // }); return Ok(self.tx_id) } @@ -410,25 +410,10 @@ where TBackend: TransactionBackend + 'static } else if response.rejection_reason == TxSubmissionRejectionReason::AlreadyMined { info!( target: LOG_TARGET, - "Transaction (TxId: {}) is Already Mined according to Base Node.", self.tx_id + "Transaction (TxId: {}) is Already Mined according to Base Node. Will be completed by transaction \ + validation protocol.", + self.tx_id ); - self.resources - .db - .mine_completed_transaction(self.tx_id) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionMined(self.tx_id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); } else { info!( target: LOG_TARGET, @@ -497,56 +482,70 @@ where TBackend: TransactionBackend + 'static // Mined? if response.location == TxLocation::Mined { - self.resources - .db - .set_transaction_confirmations(self.tx_id, response.confirmations) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - - self.resources - .db - .set_transaction_mined_height( - self.tx_id, - response.height_of_longest_chain.saturating_sub(response.confirmations), - ) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - - if response.confirmations >= self.resources.config.num_confirmations_required as u64 { - info!( - target: LOG_TARGET, - "Transaction (TxId: {}) detected as mined and CONFIRMED with {} confirmations", - self.tx_id, - response.confirmations - ); - return Ok(true); - } info!( target: LOG_TARGET, - "Transaction (TxId: {}) detected as mined but UNCONFIRMED with {} confirmations", - self.tx_id, - response.confirmations + "Broadcast transaction detected as mined, will be managed by transaction validation protoocol" ); - self.resources - .db - .mine_completed_transaction(self.tx_id) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionMinedUnconfirmed( - self.tx_id, - response.confirmations, - ))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); + return Ok(true); + // self.resources + // .db + // .set_transaction_confirmations(self.tx_id, response.confirmations) + // .await + // .for_protocol(self.tx_id)?; + // + // self.resources + // .db + // .set_transaction_mined_height( + // self.tx_id, + // response.height_of_longest_chain.saturating_sub(response.confirmations), + // response + // .block_hash + // .map(|bh| bh.clone()) + // .ok_or_else(|| { + // TransactionServiceError::InvalidMessageError( + // "Block hash was not provided for mined transaction".to_string(), + // ) + // }) + // .for_protocol(self.tx_id)?, + // ) + // .await + // .for_protocol(self.tx_id)?; + // + // if response.confirmations >= self.resources.config.num_confirmations_required as u64 { + // info!( + // target: LOG_TARGET, + // "Transaction (TxId: {}) detected as mined and CONFIRMED with {} confirmations", + // self.tx_id, + // response.confirmations + // ); + // return Ok(true); + // } + // info!( + // target: LOG_TARGET, + // "Transaction (TxId: {}) detected as mined but UNCONFIRMED with {} confirmations", + // self.tx_id, + // response.confirmations + // ); + // self.resources + // .db + // .mine_completed_transaction(self.tx_id) + // .await + // .for_protocol(self.tx_id)?; + // let _ = self + // .resources + // .event_publisher + // .send(Arc::new(TransactionEvent::TransactionMinedUnconfirmed( + // self.tx_id, + // response.confirmations, + // ))) + // .map_err(|e| { + // trace!( + // target: LOG_TARGET, + // "Error sending event because there are no subscribers: {:?}", + // e + // ); + // e + // }); } else if response.location != TxLocation::InMempool { if !self.first_rejection { info!( diff --git a/base_layer/wallet/src/transaction_service/protocols/transaction_coinbase_monitoring_protocol.rs b/base_layer/wallet/src/transaction_service/protocols/transaction_coinbase_monitoring_protocol.rs index 65fb58f601..6d4e31b233 100644 --- a/base_layer/wallet/src/transaction_service/protocols/transaction_coinbase_monitoring_protocol.rs +++ b/base_layer/wallet/src/transaction_service/protocols/transaction_coinbase_monitoring_protocol.rs @@ -23,7 +23,7 @@ use crate::{ output_manager_service::TxId, transaction_service::{ - error::{TransactionServiceError, TransactionServiceProtocolError}, + error::{TransactionServiceError, TransactionServiceProtocolError, TransactionServiceProtocolErrorExt}, handle::TransactionEvent, service::TransactionServiceResources, storage::{database::TransactionBackend, models::CompletedTransaction}, @@ -452,7 +452,7 @@ where TBackend: TransactionBackend + 'static async fn query_coinbase_transaction( &mut self, signature: Signature, - completed_tx: CompletedTransaction, + _completed_tx: CompletedTransaction, client: &mut BaseNodeWalletRpcClient, ) -> Result<(bool, Option), TransactionServiceProtocolError> { trace!( @@ -483,56 +483,56 @@ where TBackend: TransactionBackend + 'static }, }; - if !(response.is_synced || - response.location == TxLocation::Mined && - response.confirmations >= self.resources.config.num_confirmations_required) - { - info!( - target: LOG_TARGET, - "Base Node reports not being synced, coinbase monitoring will be retried." - ); - return Ok((false, Some(response.height_of_longest_chain))); - } + // if !(response.is_synced || + // response.location == TxLocation::Mined && + // response.confirmations >= self.resources.config.num_confirmations_required) + // { + // info!( + // target: LOG_TARGET, + // "Base Node reports not being synced, coinbase monitoring will be retried." + // ); + // return Ok((false, Some(response.height_of_longest_chain))); + // } // Mined? if response.location == TxLocation::Mined { - if response.confirmations >= self.resources.config.num_confirmations_required { - info!( - target: LOG_TARGET, - "Coinbase transaction (TxId: {}) detected as mined and CONFIRMED with {} confirmations", - self.tx_id, - response.confirmations - ); - self.resources - .output_manager_service - .confirm_transaction( - self.tx_id, - completed_tx.transaction.body.inputs().clone(), - completed_tx.transaction.body.outputs().clone(), - ) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - - self.resources - .db - .confirm_broadcast_or_coinbase_transaction(self.tx_id) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionMined(self.tx_id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); - return Ok((true, Some(response.height_of_longest_chain))); - } + // if response.confirmations >= self.resources.config.num_confirmations_required { + // info!( + // target: LOG_TARGET, + // "Coinbase transaction (TxId: {}) detected as mined and CONFIRMED with {} confirmations", + // self.tx_id, + // response.confirmations + // ); + // self.resources + // .output_manager_service + // .confirm_transaction( + // self.tx_id, + // completed_tx.transaction.body.inputs().clone(), + // completed_tx.transaction.body.outputs().clone(), + // ) + // .await + // .for_protocol(self.tx_id)?; + // + // self.resources + // .db + // .confirm_broadcast_or_coinbase_transaction(self.tx_id) + // .await + // .for_protocol(self.tx_id)?; + // + // let _ = self + // .resources + // .event_publisher + // .send(Arc::new(TransactionEvent::TransactionMined(self.tx_id))) + // .map_err(|e| { + // trace!( + // target: LOG_TARGET, + // "Error sending event because there are no subscribers: {:?}", + // e + // ); + // e + // }); + // return Ok((true, Some(response.height_of_longest_chain))); + // } info!( target: LOG_TARGET, "Coinbase transaction (TxId: {}) detected as mined but UNCONFIRMED with {} confirmations", @@ -542,22 +542,34 @@ where TBackend: TransactionBackend + 'static self.resources .db - .set_transaction_mined_height(self.tx_id, self.block_height) - .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; - self.resources - .db - .mine_completed_transaction(self.tx_id) + .set_transaction_mined_height( + self.tx_id, + true, + self.block_height, + response + .block_hash + .clone() + .ok_or_else(|| { + TransactionServiceError::InvalidMessageError( + "Missing block hash for mined transaction".to_string(), + ) + }) + .for_protocol(self.tx_id)?, + response.confirmations, + false // Will be picked up in + // response.confirmations >= self.resources.config.num_confirmations_required, + ) .await - .map_err(|e| TransactionServiceProtocolError::new(self.tx_id, TransactionServiceError::from(e)))?; + .for_protocol(self.tx_id)?; let _ = self .resources .event_publisher - .send(Arc::new(TransactionEvent::TransactionMinedUnconfirmed( - self.tx_id, - response.confirmations, - ))) + .send(Arc::new(TransactionEvent::TransactionMinedUnconfirmed { + tx_id: self.tx_id, + num_confirmations: response.confirmations, + is_valid: true, + })) .map_err(|e| { trace!( target: LOG_TARGET, @@ -566,6 +578,8 @@ where TBackend: TransactionBackend + 'static ); e }); + + return Ok((true, Some(response.height_of_longest_chain))); } else if response.location == TxLocation::InMempool { debug!( target: LOG_TARGET, diff --git a/base_layer/wallet/src/transaction_service/protocols/transaction_validation_protocol.rs b/base_layer/wallet/src/transaction_service/protocols/transaction_validation_protocol.rs deleted file mode 100644 index d0b2f7f6ac..0000000000 --- a/base_layer/wallet/src/transaction_service/protocols/transaction_validation_protocol.rs +++ /dev/null @@ -1,589 +0,0 @@ -// Copyright 2020. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use crate::{ - transaction_service::{ - error::{TransactionServiceError, TransactionServiceProtocolError}, - handle::TransactionEvent, - service::TransactionServiceResources, - storage::{ - database::TransactionBackend, - models::{CompletedTransaction, TransactionStatus}, - }, - }, - types::ValidationRetryStrategy, -}; -use futures::{FutureExt, StreamExt}; -use log::*; -use std::{cmp, convert::TryFrom, sync::Arc, time::Duration}; -use tari_comms::{peer_manager::NodeId, types::CommsPublicKey, PeerConnection}; -use tari_core::{ - base_node::{ - proto::wallet_rpc::{TxLocation, TxQueryBatchResponse}, - rpc::BaseNodeWalletRpcClient, - }, - proto::{base_node::Signatures as SignaturesProto, types::Signature as SignatureProto}, -}; -use tokio::{sync::broadcast, time::delay_for}; - -const LOG_TARGET: &str = "wallet::transaction_service::protocols::validation_protocol"; - -pub struct TransactionValidationProtocol -where TBackend: TransactionBackend + 'static -{ - id: u64, - resources: TransactionServiceResources, - timeout: Duration, - base_node_public_key: CommsPublicKey, - base_node_update_receiver: Option>, - timeout_update_receiver: Option>, - retry_strategy: ValidationRetryStrategy, - base_node_synced: bool, -} - -/// This protocol will check all of the mined transactions (both valid and invalid) in the db to see if they are present -/// on the current base node. # Behaviour -/// - If a valid transaction is not present the protocol will mark the transaction as invalid -/// - If an invalid transaction is present on th ebase node it will be marked as valid -/// - If a Confirmed mined transaction is present but no longer confirmed its status will change to MinedUnconfirmed -impl TransactionValidationProtocol -where TBackend: TransactionBackend + 'static -{ - pub fn new( - id: u64, - resources: TransactionServiceResources, - base_node_public_key: CommsPublicKey, - timeout: Duration, - base_node_update_receiver: broadcast::Receiver, - timeout_update_receiver: broadcast::Receiver, - retry_strategy: ValidationRetryStrategy, - ) -> Self { - Self { - id, - resources, - timeout, - base_node_public_key, - base_node_update_receiver: Some(base_node_update_receiver), - timeout_update_receiver: Some(timeout_update_receiver), - retry_strategy, - base_node_synced: true, - } - } - - /// The task that defines the execution of the protocol. - pub async fn execute(mut self) -> Result { - let mut timeout_update_receiver = self - .timeout_update_receiver - .take() - .ok_or_else(|| TransactionServiceProtocolError::new(self.id, TransactionServiceError::InvalidStateError))? - .fuse(); - - let mut base_node_update_receiver = self - .base_node_update_receiver - .take() - .ok_or_else(|| TransactionServiceProtocolError::new(self.id, TransactionServiceError::InvalidStateError))? - .fuse(); - - let mut shutdown = self.resources.shutdown_signal.clone(); - - let total_retries_str = match self.retry_strategy { - ValidationRetryStrategy::Limited(n) => format!("{}", n), - ValidationRetryStrategy::UntilSuccess => "∞".to_string(), - }; - - info!( - "Starting Transaction Validation Protocol (Id: {}) with {} retries", - self.id, total_retries_str - ); - - let mut batches = self - .get_transaction_batches() - .await - .map_err(|e| TransactionServiceProtocolError::new(self.id, e))?; - let mut retries = 0; - - // Main protocol loop - 'main: loop { - if let ValidationRetryStrategy::Limited(max_retries) = self.retry_strategy { - if retries > max_retries { - info!( - target: LOG_TARGET, - "Maximum attempts exceeded for Transaction Validation Protocol (Id: {})", self.id - ); - // If this retry is not because of a !base_node_synced then we emit this error event, if the retries - // are due to a base node NOT being synced then we rely on the TransactionValidationDelayed event - // because we were actually able to connect - if self.base_node_synced { - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionValidationFailure(self.id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); - } - return Err(TransactionServiceProtocolError::new( - self.id, - TransactionServiceError::MaximumAttemptsExceeded, - )); - } - } - // Assume base node is synced until we achieve a connection and it tells us it is not synced - self.base_node_synced = true; - - let base_node_node_id = NodeId::from_key(&self.base_node_public_key); - let mut connection: Option = None; - - let delay = delay_for(self.timeout); - - debug!( - target: LOG_TARGET, - "Connecting to Base Node (Public Key: {})", self.base_node_public_key, - ); - futures::select! { - dial_result = self.resources.connectivity_manager.dial_peer(base_node_node_id.clone()).fuse() => { - match dial_result { - Ok(base_node_connection) => { - connection = Some(base_node_connection); - }, - Err(e) => { - info!(target: LOG_TARGET, "Problem connecting to base node: {} for Transaction Validation Protocol", e); - }, - } - }, - new_base_node = base_node_update_receiver.select_next_some() => { - - match new_base_node { - Ok(_) => { - info!(target: LOG_TARGET, "Aborting Transaction Validation Protocol as new Base node is set"); - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionValidationAborted(self.id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); - return Ok(self.id); - }, - Err(e) => { - trace!( - target: LOG_TARGET, - "Transaction Validation protocol event 'base_node_update' triggered with error: {:?}", - - e, - ); - } - } - } - updated_timeout = timeout_update_receiver.select_next_some() => { - match updated_timeout { - Ok(to) => { - self.timeout = to; - info!( - target: LOG_TARGET, - "Transaction Validation protocol timeout updated to {:?}", self.timeout - ); - }, - Err(e) => { - trace!( - target: LOG_TARGET, - "Transaction Validation protocol event 'updated_timeout' triggered with error: {:?}", - - e, - ); - } - } - }, - _ = shutdown => { - info!(target: LOG_TARGET, "Transaction Validation Protocol shutting down because it received the shutdown signal"); - return Err(TransactionServiceProtocolError::new(self.id, TransactionServiceError::Shutdown)) - }, - } - - let mut base_node_connection = match connection { - None => { - futures::select! { - _ = delay.fuse() => { - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionValidationTimedOut(self.id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event {:?}, because there are no subscribers.", - e.0 - ); - e - }); - retries += 1; - continue; - }, - _ = shutdown => { - info!(target: LOG_TARGET, "Transaction Validation Protocol shutting down because it received the shutdown signal"); - return Err(TransactionServiceProtocolError::new(self.id, TransactionServiceError::Shutdown)) - }, - } - }, - Some(c) => c, - }; - - let mut client = match base_node_connection - .connect_rpc_using_builder( - BaseNodeWalletRpcClient::builder() - .with_deadline(self.timeout) - .with_handshake_timeout(self.timeout), - ) - .await - { - Ok(c) => c, - Err(e) => { - warn!(target: LOG_TARGET, "Problem establishing RPC connection: {}", e); - delay.await; - retries += 1; - continue; - }, - }; - - debug!(target: LOG_TARGET, "RPC client connected"); - - 'per_tx: loop { - let batch = if let Some(b) = batches.pop() { - b - } else { - break 'main; - }; - let delay = delay_for(self.timeout); - futures::select! { - new_base_node = base_node_update_receiver.select_next_some() => { - match new_base_node { - Ok(_) => { - info!(target: LOG_TARGET, "Aborting Transaction Validation Protocol as new Base node is set"); - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionValidationAborted(self.id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); - return Ok(self.id); - }, - Err(e) => { - trace!( - target: LOG_TARGET, - "Transaction Validation protocol event 'base_node_update' triggered with error: {:?}", - - e, - ); - } - } - }, - result = self.transaction_query_batch(batch.clone(), &mut client).fuse() => { - match result { - Ok(synced) => { - self.base_node_synced = synced; - if !synced { - info!(target: LOG_TARGET, "Base Node reports not being synced, will retry."); - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionValidationDelayed(self.id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); - delay.await; - retries += 1; - batches = self.get_transaction_batches().await.map_err(|e| TransactionServiceProtocolError::new(self.id, e))?; - break 'per_tx; - } - }, - Err(TransactionServiceError::RpcError(e)) => { - warn!(target: LOG_TARGET, "Error with RPC Client: {}. Retrying RPC client connection.", e); - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionValidationTimedOut(self.id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event {:?}, because there are no subscribers.", - e.0 - ); - e - }); - delay.await; - batches.push(batch); - retries += 1; - break 'per_tx; - } - Err(e) => { - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionValidationFailure(self.id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); - return Err(TransactionServiceProtocolError::new(self.id,e)); - }, - } - }, - updated_timeout = timeout_update_receiver.select_next_some() => { - match updated_timeout { - Ok(to) => { - self.timeout = to; - info!( - target: LOG_TARGET, - "Transaction Validation protocol timeout updated to {:?}", self.timeout - ); - }, - Err(e) => { - trace!( - target: LOG_TARGET, - "Transaction Validation protocol event 'updated_timeout' triggered with error: {:?}", - - e, - ); - } - } - }, - _ = shutdown => { - info!(target: LOG_TARGET, "Transaction Validation Protocol shutting down because it received the shutdown signal"); - return Err(TransactionServiceProtocolError::new(self.id, TransactionServiceError::Shutdown)) - }, - } - } - } - - let _ = self - .resources - .event_publisher - .send(Arc::new(TransactionEvent::TransactionValidationSuccess(self.id))) - .map_err(|e| { - trace!( - target: LOG_TARGET, - "Error sending event because there are no subscribers: {:?}", - e - ); - e - }); - - Ok(self.id) - } - - /// Attempt to query the location of the transaction from the base node via RPC. - /// # Returns: - /// `Ok(true)` => Transaction was successfully mined and confirmed - /// `Ok(false)` => There was a problem with the RPC call or the transaction is not mined but still in the mempool - /// and this should be retried `Err(_)` => The transaction was rejected by the base node and the protocol should - /// end. - async fn transaction_query_batch( - &mut self, - batch: Vec, - client: &mut BaseNodeWalletRpcClient, - ) -> Result { - let mut batch_signatures = Vec::new(); - for tx in batch.iter() { - let signature = tx - .transaction - .first_kernel_excess_sig() - .ok_or(TransactionServiceError::InvalidTransaction)?; - batch_signatures.push(SignatureProto::from(signature.clone())); - } - - let batch_response = client - .transaction_batch_query(SignaturesProto { sigs: batch_signatures }) - .await?; - - if !batch_response.is_synced { - return Ok(false); - } - - for response_proto in batch_response.responses { - let response = TxQueryBatchResponse::try_from(response_proto) - .map_err(TransactionServiceError::ProtobufConversionError)?; - - if let Some(queried_tx) = batch.iter().find(|tx| { - if let Some(sig) = tx.transaction.first_kernel_excess_sig() { - sig == &response.signature - } else { - false - } - }) { - // Mined? - if response.location == TxLocation::Mined { - if !queried_tx.valid { - info!( - target: LOG_TARGET, - "Transaction (TxId: {}) is VALID according to base node, status will be updated", - queried_tx.tx_id - ); - if let Err(e) = self - .resources - .db - .set_completed_transaction_validity(queried_tx.tx_id, true) - .await - { - warn!( - target: LOG_TARGET, - "Error setting transaction (TxId: {}) validity: {}", queried_tx.tx_id, e - ); - } - } - if response.confirmations >= self.resources.config.num_confirmations_required as u64 { - if queried_tx.status == TransactionStatus::MinedUnconfirmed { - info!( - target: LOG_TARGET, - "Transaction (TxId: {}) is MINED and CONFIRMED according to base node, status will be \ - updated", - queried_tx.tx_id - ); - if let Err(e) = self - .resources - .db - .confirm_broadcast_or_coinbase_transaction(queried_tx.tx_id) - .await - { - warn!( - target: LOG_TARGET, - "Error confirming mined transaction (TxId: {}): {}", queried_tx.tx_id, e - ); - } - if let Err(e) = self - .resources - .output_manager_service - .confirm_transaction( - queried_tx.tx_id, - queried_tx.transaction.body.inputs().clone(), - queried_tx.transaction.body.outputs().clone(), - ) - .await - { - debug!( - target: LOG_TARGET, - "Error confirming outputs transaction (TxId: {}) that was validated with new base \ - node: {}. Usually means this transaction was confirmed in the past", - queried_tx.tx_id, - e - ); - } - } - } else if queried_tx.status == TransactionStatus::MinedConfirmed { - info!( - target: LOG_TARGET, - "Transaction (TxId: {}) is MINED but UNCONFIRMED according to base node, status will be \ - updated", - queried_tx.tx_id - ); - if let Err(e) = self.resources.db.unconfirm_mined_transaction(queried_tx.tx_id).await { - warn!( - target: LOG_TARGET, - "Error unconfirming mined transaction (TxId: {}): {}", queried_tx.tx_id, e - ); - } - } - } else if queried_tx.valid { - info!( - target: LOG_TARGET, - "Transaction (TxId: {}) is INVALID according to base node, status will be updated", - queried_tx.tx_id - ); - if let Err(e) = self - .resources - .db - .set_completed_transaction_validity(queried_tx.tx_id, false) - .await - { - warn!( - target: LOG_TARGET, - "Error setting transaction (TxId: {}) validity: {}", queried_tx.tx_id, e - ); - } - } - } else { - debug!( - target: LOG_TARGET, - "Could not find transaction corresponding to returned query response" - ); - } - } - Ok(true) - } - - /// Get completed transactions from db and sort the mined transactions into batches - async fn get_transaction_batches(&self) -> Result>, TransactionServiceError> { - let mut completed_txs: Vec = self - .resources - .db - .get_completed_transactions() - .await? - .values() - .filter(|tx| { - tx.status == TransactionStatus::MinedUnconfirmed || tx.status == TransactionStatus::MinedConfirmed - }) - .cloned() - .collect(); - // Determine how many rounds of base node request we need to query all the transactions in batches of - // max_tx_query_batch_size - let num_batches = - ((completed_txs.len() as f32) / (self.resources.config.max_tx_query_batch_size as f32 + 0.1)) as usize + 1; - - let mut batches: Vec> = Vec::new(); - for _b in 0..num_batches { - let mut batch = Vec::new(); - for tx in - completed_txs.drain(..cmp::min(self.resources.config.max_tx_query_batch_size, completed_txs.len())) - { - batch.push(tx); - } - if !batch.is_empty() { - batches.push(batch); - } - } - Ok(batches) - } -} diff --git a/base_layer/wallet/src/transaction_service/protocols/transaction_validation_protocol_v2.rs b/base_layer/wallet/src/transaction_service/protocols/transaction_validation_protocol_v2.rs new file mode 100644 index 0000000000..53fdafdb95 --- /dev/null +++ b/base_layer/wallet/src/transaction_service/protocols/transaction_validation_protocol_v2.rs @@ -0,0 +1,410 @@ +// Copyright 2021. The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use crate::transaction_service::{ + config::TransactionServiceConfig, + error::{TransactionServiceError, TransactionServiceProtocolError, TransactionServiceProtocolErrorExt}, + handle::{TransactionEvent, TransactionEventSender}, + storage::{ + database::{TransactionBackend, TransactionDatabase}, + models::CompletedTransaction, + }, +}; +use log::*; +use std::{ + collections::HashMap, + convert::{TryFrom, TryInto}, + sync::Arc, +}; +use tari_common_types::types::BlockHash; +use tari_comms::{ + connectivity::ConnectivityRequester, + protocol::rpc::{RpcError::RequestFailed, RpcStatusCode::NotFound}, + types::CommsPublicKey, +}; +use tari_core::{ + base_node::{ + proto::wallet_rpc::{TxLocation, TxQueryBatchResponse}, + rpc::BaseNodeWalletRpcClient, + sync::rpc::BaseNodeSyncRpcClient, + }, + blocks::BlockHeader, + proto::{base_node::Signatures as SignaturesProto, types::Signature as SignatureProto}, +}; +use tari_crypto::tari_utilities::{hex::Hex, Hashable}; + +const LOG_TARGET: &str = "wallet::transaction_service::protocols::validation_protocol_v2"; + +pub struct TransactionValidationProtocolV2 { + db: TransactionDatabase, + base_node_pk: CommsPublicKey, + operation_id: u64, + batch_size: usize, + connectivity_requester: ConnectivityRequester, + config: TransactionServiceConfig, + event_publisher: TransactionEventSender, +} + +#[allow(unused_variables)] +impl TransactionValidationProtocolV2 { + pub fn new( + db: TransactionDatabase, + base_node_pk: CommsPublicKey, + connectivity_requester: ConnectivityRequester, + config: TransactionServiceConfig, + event_publisher: TransactionEventSender, + ) -> Self { + Self { + operation_id: 122, // Get a real tx id + db, + batch_size: 10, + base_node_pk, + connectivity_requester, + config, + event_publisher, + } + } + + pub async fn execute(mut self) -> Result { + let mut base_node_connection = self + .connectivity_requester + .dial_peer(self.base_node_pk.clone().into()) + .await + .for_protocol(self.operation_id)?; + let mut client = base_node_connection + .connect_rpc_using_builder( + BaseNodeSyncRpcClient::builder().with_deadline(self.config.chain_monitoring_timeout), + ) + .await + .for_protocol(self.operation_id)?; + + let mut base_node_wallet_client = base_node_connection + .connect_rpc_using_builder( + BaseNodeWalletRpcClient::builder().with_deadline(self.config.chain_monitoring_timeout), + ) + .await + .for_protocol(self.operation_id)?; + + self.check_for_reorgs(&mut client).await?; + info!( + target: LOG_TARGET, + "Checking if transactions have been mined since last we checked" + ); + let unmined_transactions = self + .db + .fetch_unmined_transactions() + .await + .for_protocol(self.operation_id) + .unwrap(); + for batch in unmined_transactions.chunks(self.batch_size) { + let (mined, unmined, tip_info) = self + .query_base_node_for_transactions(batch, &mut base_node_wallet_client) + .await + .for_protocol(self.operation_id)?; + info!( + target: LOG_TARGET, + "Base node returned {} as mined and {} as unmined", + mined.len(), + unmined.len() + ); + for (tx, mined_height, mined_in_block, num_confirmations) in &mined { + info!(target: LOG_TARGET, "Updating transaction {} as mined", tx.tx_id); + self.update_transaction_as_mined(tx, mined_in_block, *mined_height, *num_confirmations) + .await?; + } + if let Some((tip_height, tip_block)) = tip_info { + for tx in &unmined { + // Treat coinbases separately + if tx.is_coinbase_transaction() { + if tx.coinbase_block_height.unwrap_or_default() <= tip_height { + info!(target: LOG_TARGET, "Updated coinbase {} as mined invalid", tx.tx_id); + self.update_coinbase_as_lost( + tx, + &tip_block, + tip_height, + tip_height.saturating_sub(tx.coinbase_block_height.unwrap_or_default()), + ) + .await?; + } else { + info!( + target: LOG_TARGET, + "Coinbase not found, but it is for a block that is not yet in the chain. Coinbase \ + height: {}, tip height:{}", + tx.coinbase_block_height.unwrap_or_default(), + tip_height + ); + } + } else { + info!(target: LOG_TARGET, "Updated transaction {} as unmined", tx.tx_id); + self.update_transaction_as_unmined(&tx).await?; + } + } + } + } + self.publish_event(TransactionEvent::TransactionValidationSuccess(self.operation_id)); + Ok(self.operation_id) + } + + fn publish_event(&self, event: TransactionEvent) { + if let Err(e) = self.event_publisher.send(Arc::new(event)) { + debug!( + target: LOG_TARGET, + "Error sending event because there are no subscribers: {:?}", e + ); + } + } + + async fn check_for_reorgs( + &mut self, + client: &mut BaseNodeSyncRpcClient, + ) -> Result<(), TransactionServiceProtocolError> { + info!( + target: LOG_TARGET, + "Checking last mined transactions to see if the base node has re-orged" + ); + while let Some(last_mined_transaction) = self + .db + .get_last_mined_transaction() + .await + .for_protocol(self.operation_id) + .unwrap() + { + let mined_height = last_mined_transaction.mined_height.unwrap(); // TODO: fix unwrap + let mined_in_block_hash = last_mined_transaction.mined_in_block.clone().unwrap(); // TODO: fix unwrap. + let block_at_height = self + .get_base_node_block_at_height(mined_height, client) + .await + .for_protocol(self.operation_id)?; + if block_at_height.is_none() || block_at_height.unwrap() != mined_in_block_hash { + // Chain has reorged since we last + warn!( + target: LOG_TARGET, + "The block that transaction (excess:{}) was in has been reorged out, will try to find this \ + transaction again, but these funds have potentially been re-orged out of the chain", + last_mined_transaction + .transaction + .body + .kernels() + .first() + .map(|k| k.excess.to_hex()) + .unwrap() + ); + self.update_transaction_as_unmined(&last_mined_transaction).await?; + } else { + info!( + target: LOG_TARGET, + "Last mined transaction is still in the block chain according to base node." + ); + break; + } + } + Ok(()) + } + + async fn query_base_node_for_transactions( + &self, + batch: &[CompletedTransaction], + base_node_client: &mut BaseNodeWalletRpcClient, + ) -> Result< + ( + Vec<(CompletedTransaction, u64, BlockHash, u64)>, + Vec, + Option<(u64, BlockHash)>, + ), + TransactionServiceError, + > { + let mut mined = vec![]; + let mut unmined = vec![]; + + let mut batch_signatures = HashMap::new(); + for tx in batch.iter() { + // Imported transactions do not have a signature + if let Some(sig) = tx.transaction.first_kernel_excess_sig() { + batch_signatures.insert(sig.clone(), tx); + } + } + + if batch_signatures.is_empty() { + info!(target: LOG_TARGET, "No transactions needed to query with the base node"); + return Ok((mined, unmined, None)); + } + + info!( + target: LOG_TARGET, + "Asking base node for location of {} transactions by excess", + batch.len() + ); + + let batch_response = base_node_client + .transaction_batch_query(SignaturesProto { + sigs: batch_signatures + .keys() + .map(|s| SignatureProto::from(s.clone())) + .collect(), + }) + .await?; + + for response_proto in batch_response.responses { + let response = TxQueryBatchResponse::try_from(response_proto) + .map_err(TransactionServiceError::ProtobufConversionError)?; + let sig = response.signature; + if let Some(completed_tx) = batch_signatures.get(&sig) { + if response.location == TxLocation::Mined { + mined.push(( + (*completed_tx).clone(), + response.block_height, + response.block_hash.unwrap(), + response.confirmations, + )); + } else { + unmined.push((*completed_tx).clone()); + } + } + } + + Ok(( + mined, + unmined, + Some(( + batch_response.height_of_longest_chain, + batch_response.tip_hash.ok_or_else(|| { + TransactionServiceError::ProtobufConversionError("Missing `tip_hash` field".to_string()) + })?, + )), + )) + } + + async fn get_base_node_block_at_height( + &mut self, + height: u64, + client: &mut BaseNodeSyncRpcClient, + ) -> Result, TransactionServiceError> { + let result = match client.get_header_by_height(height).await { + Ok(r) => r, + Err(rpc_error) => { + warn!(target: LOG_TARGET, "Error asking base node for header:{}", rpc_error); + match &rpc_error { + RequestFailed(status) => { + if status.status_code() == NotFound { + return Ok(None); + } else { + return Err(rpc_error.into()); + } + }, + _ => { + return Err(rpc_error.into()); + }, + } + }, + }; + + let block_header: BlockHeader = result.try_into().map_err(|s| { + TransactionServiceError::InvalidMessageError(format!("Could not convert block header: {}", s)) + })?; + Ok(Some(block_header.hash())) + } + + #[allow(clippy::ptr_arg)] + async fn update_transaction_as_mined( + &self, + tx: &CompletedTransaction, + mined_in_block: &BlockHash, + mined_height: u64, + num_confirmations: u64, + ) -> Result<(), TransactionServiceProtocolError> { + self.db + .set_transaction_mined_height( + tx.tx_id, + true, + mined_height, + mined_in_block.clone(), + num_confirmations, + num_confirmations >= self.config.num_confirmations_required, + ) + .await + .for_protocol(self.operation_id)?; + + if num_confirmations >= self.config.num_confirmations_required { + self.publish_event(TransactionEvent::TransactionMined { + tx_id: tx.tx_id, + is_valid: true, + }) + } else { + self.publish_event(TransactionEvent::TransactionMinedUnconfirmed { + tx_id: tx.tx_id, + num_confirmations, + is_valid: true, + }) + } + + Ok(()) + } + + #[allow(clippy::ptr_arg)] + async fn update_coinbase_as_lost( + &self, + tx: &CompletedTransaction, + mined_in_block: &BlockHash, + mined_height: u64, + num_confirmations: u64, + ) -> Result<(), TransactionServiceProtocolError> { + self.db + .set_transaction_mined_height( + tx.tx_id, + false, + mined_height, + mined_in_block.clone(), + num_confirmations, + num_confirmations >= self.config.num_confirmations_required, + ) + .await + .for_protocol(self.operation_id)?; + + if num_confirmations >= self.config.num_confirmations_required { + self.publish_event(TransactionEvent::TransactionMined { + tx_id: tx.tx_id, + is_valid: false, + }) + } else { + self.publish_event(TransactionEvent::TransactionMinedUnconfirmed { + tx_id: tx.tx_id, + num_confirmations, + is_valid: false, + }) + } + + Ok(()) + } + + async fn update_transaction_as_unmined( + &self, + tx: &CompletedTransaction, + ) -> Result<(), TransactionServiceProtocolError> { + self.db + .set_transaction_as_unmined(tx.tx_id) + .await + .for_protocol(self.operation_id)?; + + self.publish_event(TransactionEvent::TransactionBroadcast(tx.tx_id)); + Ok(()) + } +} diff --git a/base_layer/wallet/src/transaction_service/service.rs b/base_layer/wallet/src/transaction_service/service.rs index c30fb7f412..f989127cd1 100644 --- a/base_layer/wallet/src/transaction_service/service.rs +++ b/base_layer/wallet/src/transaction_service/service.rs @@ -28,10 +28,9 @@ use crate::{ handle::{TransactionEvent, TransactionEventSender, TransactionServiceRequest, TransactionServiceResponse}, protocols::{ transaction_broadcast_protocol::TransactionBroadcastProtocol, - transaction_coinbase_monitoring_protocol::TransactionCoinbaseMonitoringProtocol, transaction_receive_protocol::{TransactionReceiveProtocol, TransactionReceiveProtocolStage}, transaction_send_protocol::{TransactionSendProtocol, TransactionSendProtocolStage}, - transaction_validation_protocol::TransactionValidationProtocol, + transaction_validation_protocol_v2::TransactionValidationProtocolV2, }, storage::{ database::{TransactionBackend, TransactionDatabase}, @@ -85,7 +84,7 @@ use tari_crypto::{keys::DiffieHellmanSharedSecret, script, tari_utilities::ByteA use tari_p2p::domain_message::DomainMessage; use tari_service_framework::{reply_channel, reply_channel::Receiver}; use tari_shutdown::ShutdownSignal; -use tokio::{sync::broadcast, task::JoinHandle}; +use tokio::{sync::broadcast, task::JoinHandle, time::interval_at}; const LOG_TARGET: &str = "wallet::transaction_service::service"; @@ -274,9 +273,22 @@ where JoinHandle>, > = FuturesUnordered::new(); + // Should probably be block time or at least configurable + // Start a bit later so that the wallet can start up + let mut tx_validation_interval = interval_at( + (Instant::now() + Duration::from_secs(30)).into(), + Duration::from_secs(30), + ) + .fuse(); + info!(target: LOG_TARGET, "Transaction Service started"); loop { futures::select! { + // tx validation timer + _ = tx_validation_interval.select_next_some() => { + self.start_transaction_validation_protocol(ValidationRetryStrategy::Limited(0), &mut transaction_validation_protocol_handles).await?; + }, + //Incoming request request_context = request_stream.select_next_some() => { // TODO: Remove time measurements; this is to aid in system testing only @@ -638,10 +650,10 @@ where .start_transaction_validation_protocol(retry_strategy, transaction_validation_join_handles) .await .map(TransactionServiceResponse::ValidationStarted), - TransactionServiceRequest::SetCompletedTransactionValidity(tx_id, validity) => self - .set_completed_transaction_validity(tx_id, validity) - .await - .map(|_| TransactionServiceResponse::CompletedTransactionValidityChanged), + /* TransactionServiceRequest::SetCompletedTransactionValidity(tx_id, validity) => self + * .set_completed_transaction_validity(tx_id, validity) + * .await + * .map(|_| TransactionServiceResponse::CompletedTransactionValidityChanged), */ }; // If the individual handlers did not already send the API response then do it here. @@ -1100,18 +1112,18 @@ where Ok(()) } - async fn set_completed_transaction_validity( - &mut self, - tx_id: TxId, - valid: bool, - ) -> Result<(), TransactionServiceError> { - self.resources - .db - .set_completed_transaction_validity(tx_id, valid) - .await?; - - Ok(()) - } + // async fn set_completed_transaction_validity( + // &mut self, + // tx_id: TxId, + // valid: bool, + // ) -> Result<(), TransactionServiceError> { + // self.resources + // .db + // .set_completed_transaction_validity(tx_id, valid) + // .await?; + // + // Ok(()) + // } /// Handle a Transaction Cancelled message received from the Comms layer pub async fn handle_transaction_cancelled_message( @@ -1516,7 +1528,7 @@ where async fn start_transaction_validation_protocol( &mut self, - retry_strategy: ValidationRetryStrategy, + _retry_strategy: ValidationRetryStrategy, join_handles: &mut FuturesUnordered>>, ) -> Result { if self.base_node_public_key.is_none() { @@ -1524,22 +1536,31 @@ where } trace!(target: LOG_TARGET, "Starting transaction validation protocols"); let id = OsRng.next_u64(); - let timeout = match self.power_mode { + let _timeout = match self.power_mode { PowerMode::Normal => self.config.broadcast_monitoring_timeout, PowerMode::Low => self.config.low_power_polling_timeout, }; match self.base_node_public_key.clone() { None => return Err(TransactionServiceError::NoBaseNodeKeysProvided), Some(pk) => { - let protocol = TransactionValidationProtocol::new( - id, - self.resources.clone(), + // let protocol = TransactionValidationProtocol::new( + // id, + // self.resources.clone(), + // pk, + // timeout, + // self.base_node_update_publisher.subscribe(), + // self.timeout_update_publisher.subscribe(), + // retry_strategy, + // ); + + let protocol = TransactionValidationProtocolV2::new( + self.resources.db.clone(), pk, - timeout, - self.base_node_update_publisher.subscribe(), - self.timeout_update_publisher.subscribe(), - retry_strategy, + self.resources.connectivity_manager.clone(), + self.resources.config.clone(), + self.event_publisher.clone(), ); + let join_handle = tokio::spawn(protocol.execute()); join_handles.push(join_handle); }, @@ -1624,6 +1645,12 @@ where { return Err(TransactionServiceError::InvalidCompletedTransaction); } + if completed_tx.is_coinbase_transaction() { + return Err(TransactionServiceError::AttemptedToBroadcastCoinbaseTransaction( + completed_tx.tx_id, + )); + } + let timeout = match self.power_mode { PowerMode::Normal => self.config.broadcast_monitoring_timeout, PowerMode::Low => self.config.low_power_polling_timeout, @@ -1667,7 +1694,8 @@ where if completed_tx.valid && (completed_tx.status == TransactionStatus::Completed || completed_tx.status == TransactionStatus::Broadcast || - completed_tx.status == TransactionStatus::MinedUnconfirmed) + completed_tx.status == TransactionStatus::MinedUnconfirmed) && + !completed_tx.is_coinbase_transaction() { self.broadcast_completed_transaction(completed_tx, join_handles).await?; } @@ -1936,7 +1964,7 @@ where async fn start_coinbase_transaction_monitoring_protocol( &mut self, tx_id: TxId, - join_handles: &mut FuturesUnordered>>, + _join_handles: &mut FuturesUnordered>>, ) -> Result<(), TransactionServiceError> { let completed_tx = self.db.get_completed_transaction(tx_id).await?; @@ -1944,31 +1972,35 @@ where return Err(TransactionServiceError::InvalidCompletedTransaction); } - let block_height = if let Some(bh) = completed_tx.coinbase_block_height { + let _block_height = if let Some(bh) = completed_tx.coinbase_block_height { bh } else { 0 }; - let timeout = match self.power_mode { + let _timeout = match self.power_mode { PowerMode::Normal => self.config.broadcast_monitoring_timeout, PowerMode::Low => self.config.low_power_polling_timeout, }; match self.base_node_public_key.clone() { None => return Err(TransactionServiceError::NoBaseNodeKeysProvided), - Some(pk) => { + Some(_pk) => { if self.active_coinbase_monitoring_protocols.insert(tx_id) { - let protocol = TransactionCoinbaseMonitoringProtocol::new( - completed_tx.tx_id, - block_height, - self.resources.clone(), - timeout, - pk, - self.base_node_update_publisher.subscribe(), - self.timeout_update_publisher.subscribe(), + // let protocol = TransactionCoinbaseMonitoringProtocol::new( + // completed_tx.tx_id, + // block_height, + // self.resources.clone(), + // timeout, + // pk, + // self.base_node_update_publisher.subscribe(), + // self.timeout_update_publisher.subscribe(), + // ); + // let join_handle = tokio::spawn(protocol.execute()); + // join_handles.push(join_handle); + warn!( + target: LOG_TARGET, + "Not running coinbase protocol, it will be handled by tx validation protocol" ); - let join_handle = tokio::spawn(protocol.execute()); - join_handles.push(join_handle); } else { debug!( target: LOG_TARGET, diff --git a/base_layer/wallet/src/transaction_service/storage/database.rs b/base_layer/wallet/src/transaction_service/storage/database.rs index aaad0b0618..ebc6d3ce34 100644 --- a/base_layer/wallet/src/transaction_service/storage/database.rs +++ b/base_layer/wallet/src/transaction_service/storage/database.rs @@ -40,9 +40,11 @@ use log::*; use crate::transaction_service::storage::models::WalletTransaction; use std::{ collections::HashMap, + fmt, fmt::{Display, Error, Formatter}, sync::Arc, }; +use tari_common_types::types::BlockHash; use tari_comms::types::CommsPublicKey; use tari_core::transactions::{tari_amount::MicroTari, transaction::Transaction, types::BlindingFactor}; @@ -55,6 +57,11 @@ const LOG_TARGET: &str = "wallet::transaction_service::database"; pub trait TransactionBackend: Send + Sync + Clone { /// Retrieve the record associated with the provided DbKey fn fetch(&self, key: &DbKey) -> Result, TransactionStorageError>; + + fn fetch_last_mined_transaction(&self) -> Result, TransactionStorageError>; + + fn fetch_unmined_transactions(&self) -> Result, TransactionStorageError>; + /// Check if a record with the provided key exists in the backend. fn contains(&self, key: &DbKey) -> Result; /// Modify the state the of the backend with a write operation @@ -77,15 +84,8 @@ pub trait TransactionBackend: Send + Sync + Clone { ) -> Result<(), TransactionStorageError>; /// Indicated that a completed transaction has been broadcast to the mempools fn broadcast_completed_transaction(&self, tx_id: TxId) -> Result<(), TransactionStorageError>; - /// Indicated that a completed transaction has been detected as mined on a base node - fn mine_completed_transaction(&self, tx_id: TxId) -> Result<(), TransactionStorageError>; /// Indicated that a broadcast transaction has been detected as confirm on a base node fn confirm_broadcast_or_coinbase_transaction(&self, tx_id: TxId) -> Result<(), TransactionStorageError>; - /// Indicated that a mined transaction has been detected as unconfirmed on a base node, due to reorg or base node - /// switch - fn unconfirm_mined_transaction(&self, tx_id: TxId) -> Result<(), TransactionStorageError>; - /// Set transaction's validity - fn set_completed_transaction_validity(&self, tx_id: TxId, valid: bool) -> Result<(), TransactionStorageError>; /// Cancel Completed transaction, this will update the transaction status fn cancel_completed_transaction(&self, tx_id: TxId) -> Result<(), TransactionStorageError>; /// Set cancellation on Pending transaction, this will update the transaction status @@ -117,11 +117,26 @@ pub trait TransactionBackend: Send + Sync + Clone { fn increment_send_count(&self, tx_id: TxId) -> Result<(), TransactionStorageError>; /// Update a transactions number of confirmations fn update_confirmations(&self, tx_id: TxId, confirmations: u64) -> Result<(), TransactionStorageError>; - /// Update a transactions mined height - fn update_mined_height(&self, tx_id: TxId, mined_height: u64) -> Result<(), TransactionStorageError>; + /// Update a transactions mined height. A transaction can either be mined as valid or mined as invalid + /// A normal transaction can only be mined with valid = true, + /// A coinbase transaction can either be mined as valid = true, meaning that it is the coinbase in that block + /// or valid =false, meaning that the coinbase has been awarded to another tx, but this has been confirmed by blocks + /// The mined height and block are used to determine reorgs + fn update_mined_height( + &self, + tx_id: TxId, + is_valid: bool, + mined_height: u64, + mined_in_block: BlockHash, + num_confirmations: u64, + is_confirmed: bool, + ) -> Result<(), TransactionStorageError>; + + /// Clears the mined block and height of a transaction + fn set_transaction_as_unmined(&self, tx_id: TxId) -> Result<(), TransactionStorageError>; } -#[derive(Debug, Clone, PartialEq)] +#[derive(Clone, PartialEq)] pub enum DbKey { PendingOutboundTransaction(TxId), PendingInboundTransaction(TxId), @@ -137,6 +152,59 @@ pub enum DbKey { AnyTransaction(TxId), } +impl fmt::Debug for DbKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use DbKey::*; + // Add in i64 representatives for easy debugging in sqlite. This should probably be removed at some point + match self { + PendingOutboundTransaction(tx_id) => { + write!(f, "PendingOutboundTransaction ({}u64, {}i64)", tx_id, *tx_id as i64) + }, + PendingInboundTransaction(tx_id) => { + write!(f, "PendingInboundTransaction ({}u64, {}i64)", tx_id, *tx_id as i64) + }, + CompletedTransaction(tx_id) => { + write!(f, "CompletedTransaction ({}u64, {}i64)", tx_id, *tx_id as i64) + }, + PendingOutboundTransactions => { + write!(f, "PendingOutboundTransactions ") + }, + PendingInboundTransactions => { + write!(f, "PendingInboundTransactions") + }, + CompletedTransactions => { + write!(f, "CompletedTransactions ") + }, + CancelledPendingOutboundTransactions => { + write!(f, "CancelledPendingOutboundTransactions ") + }, + CancelledPendingInboundTransactions => { + write!(f, "CancelledPendingInboundTransactions") + }, + CancelledCompletedTransactions => { + write!(f, "CancelledCompletedTransactions") + }, + CancelledPendingOutboundTransaction(tx_id) => { + write!( + f, + "CancelledPendingOutboundTransaction ({}u64, {}i64)", + tx_id, *tx_id as i64 + ) + }, + CancelledPendingInboundTransaction(tx_id) => { + write!( + f, + "CancelledPendingInboundTransaction ({}u64, {}i64)", + tx_id, *tx_id as i64 + ) + }, + AnyTransaction(tx_id) => { + write!(f, "AnyTransaction ({}u64, {}i64)", tx_id, *tx_id as i64) + }, + } + } +} + #[derive(Debug)] pub enum DbValue { PendingOutboundTransaction(Box), @@ -358,6 +426,14 @@ where T: TransactionBackend + 'static Ok(*t) } + pub async fn get_last_mined_transaction(&self) -> Result, TransactionStorageError> { + self.db.fetch_last_mined_transaction() + } + + pub async fn fetch_unmined_transactions(&self) -> Result, TransactionStorageError> { + self.db.fetch_unmined_transactions() + } + pub async fn get_completed_transaction_cancelled_or_not( &self, tx_id: TxId, @@ -590,16 +666,6 @@ where T: TransactionBackend + 'static .and_then(|inner_result| inner_result) } - /// Indicated that the specified completed transaction has been detected as mined on the base layer - pub async fn mine_completed_transaction(&self, tx_id: TxId) -> Result<(), TransactionStorageError> { - let db_clone = self.db.clone(); - - tokio::task::spawn_blocking(move || db_clone.mine_completed_transaction(tx_id)) - .await - .map_err(|err| TransactionStorageError::BlockingTaskSpawnError(err.to_string())) - .and_then(|inner_result| inner_result) - } - pub async fn add_utxo_import_transaction( &self, tx_id: TxId, @@ -698,21 +764,9 @@ where T: TransactionBackend + 'static Ok(()) } - pub async fn unconfirm_mined_transaction(&self, tx_id: TxId) -> Result<(), TransactionStorageError> { - let db_clone = self.db.clone(); - tokio::task::spawn_blocking(move || db_clone.unconfirm_mined_transaction(tx_id)) - .await - .map_err(|err| TransactionStorageError::BlockingTaskSpawnError(err.to_string()))??; - Ok(()) - } - - pub async fn set_completed_transaction_validity( - &self, - tx_id: TxId, - valid: bool, - ) -> Result<(), TransactionStorageError> { + pub async fn set_transaction_as_unmined(&self, tx_id: TxId) -> Result<(), TransactionStorageError> { let db_clone = self.db.clone(); - tokio::task::spawn_blocking(move || db_clone.set_completed_transaction_validity(tx_id, valid)) + tokio::task::spawn_blocking(move || db_clone.set_transaction_as_unmined(tx_id)) .await .map_err(|err| TransactionStorageError::BlockingTaskSpawnError(err.to_string()))??; Ok(()) @@ -733,12 +787,25 @@ where T: TransactionBackend + 'static pub async fn set_transaction_mined_height( &self, tx_id: TxId, + is_valid: bool, mined_height: u64, + mined_in_block: BlockHash, + num_confirmations: u64, + is_confirmed: bool, ) -> Result<(), TransactionStorageError> { let db_clone = self.db.clone(); - tokio::task::spawn_blocking(move || db_clone.update_mined_height(tx_id, mined_height)) - .await - .map_err(|err| TransactionStorageError::BlockingTaskSpawnError(err.to_string()))??; + tokio::task::spawn_blocking(move || { + db_clone.update_mined_height( + tx_id, + is_valid, + mined_height, + mined_in_block, + num_confirmations, + is_confirmed, + ) + }) + .await + .map_err(|err| TransactionStorageError::BlockingTaskSpawnError(err.to_string()))??; Ok(()) } } diff --git a/base_layer/wallet/src/transaction_service/storage/models.rs b/base_layer/wallet/src/transaction_service/storage/models.rs index 37f84cc3fb..478c113850 100644 --- a/base_layer/wallet/src/transaction_service/storage/models.rs +++ b/base_layer/wallet/src/transaction_service/storage/models.rs @@ -27,6 +27,7 @@ use std::{ convert::TryFrom, fmt::{Display, Error, Formatter}, }; +use tari_common_types::types::BlockHash; use tari_comms::types::CommsPublicKey; use tari_core::transactions::{ tari_amount::MicroTari, @@ -201,6 +202,7 @@ pub struct CompletedTransaction { pub valid: bool, pub confirmations: Option, pub mined_height: Option, + pub mined_in_block: Option, } impl CompletedTransaction { @@ -236,8 +238,13 @@ impl CompletedTransaction { valid: true, confirmations: None, mined_height: None, + mined_in_block: None, } } + + pub fn is_coinbase_transaction(&self) -> bool { + self.coinbase_block_height.is_some() + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -330,6 +337,7 @@ impl From for CompletedTransaction { valid: true, confirmations: None, mined_height: None, + mined_in_block: None, } } } @@ -354,6 +362,7 @@ impl From for CompletedTransaction { valid: true, confirmations: None, mined_height: None, + mined_in_block: None, } } } diff --git a/base_layer/wallet/src/transaction_service/storage/sqlite_db.rs b/base_layer/wallet/src/transaction_service/storage/sqlite_db.rs index 0cd55fc7f2..cab4596773 100644 --- a/base_layer/wallet/src/transaction_service/storage/sqlite_db.rs +++ b/base_layer/wallet/src/transaction_service/storage/sqlite_db.rs @@ -38,7 +38,10 @@ use crate::{ }, }, }, - util::encryption::{decrypt_bytes_integral_nonce, encrypt_bytes_integral_nonce, Encryptable}, + util::{ + diesel_ext::ExpectedRowsExtension, + encryption::{decrypt_bytes_integral_nonce, encrypt_bytes_integral_nonce, Encryptable}, + }, }; use aes_gcm::{self, aead::Error as AeadError, Aes256Gcm}; use chrono::{NaiveDateTime, Utc}; @@ -46,10 +49,11 @@ use diesel::{prelude::*, result::Error as DieselError, SqliteConnection}; use log::*; use std::{ collections::HashMap, - convert::TryFrom, + convert::{TryFrom, TryInto}, str::from_utf8, sync::{Arc, MutexGuard, RwLock}, }; +use tari_common_types::types::BlockHash; use tari_comms::types::CommsPublicKey; use tari_core::transactions::{tari_amount::MicroTari, types::PublicKey}; use tari_crypto::tari_utilities::{ @@ -505,17 +509,10 @@ impl TransactionBackend for TransactionServiceSqliteDatabase { Ok(v) => { if TransactionStatus::try_from(v.status)? == TransactionStatus::Completed { v.update( - UpdateCompletedTransactionSql::from(UpdateCompletedTransaction { - status: Some(TransactionStatus::Broadcast), - timestamp: None, - cancelled: None, - direction: None, - send_count: None, - last_send_timestamp: None, - valid: None, - confirmations: None, - mined_height: None, - }), + UpdateCompletedTransactionSql { + status: Some(TransactionStatus::Broadcast as i32), + ..Default::default() + }, &(*conn), )?; } @@ -530,36 +527,6 @@ impl TransactionBackend for TransactionServiceSqliteDatabase { Ok(()) } - fn mine_completed_transaction(&self, tx_id: u64) -> Result<(), TransactionStorageError> { - let conn = self.database_connection.acquire_lock(); - - match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { - Ok(v) => { - v.update( - UpdateCompletedTransactionSql::from(UpdateCompletedTransaction { - status: Some(TransactionStatus::MinedUnconfirmed), - timestamp: None, - cancelled: None, - direction: None, - send_count: None, - last_send_timestamp: None, - valid: None, - confirmations: None, - mined_height: None, - }), - &(*conn), - )?; - }, - Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { - return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( - tx_id, - ))) - }, - Err(e) => return Err(e), - }; - Ok(()) - } - fn cancel_completed_transaction(&self, tx_id: u64) -> Result<(), TransactionStorageError> { let conn = self.database_connection.acquire_lock(); match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { @@ -779,16 +746,9 @@ impl TransactionBackend for TransactionServiceSqliteDatabase { if let Ok(tx) = CompletedTransactionSql::find(tx_id, &conn) { let update = UpdateCompletedTransactionSql { - status: None, - timestamp: None, - cancelled: None, - direction: None, - transaction_protocol: None, send_count: Some(tx.send_count + 1), last_send_timestamp: Some(Some(Utc::now().naive_utc())), - valid: None, - confirmations: None, - mined_height: None, + ..Default::default() }; tx.update(update, &conn)?; } else if let Ok(tx) = OutboundTransactionSql::find(tx_id, &conn) { @@ -816,19 +776,84 @@ impl TransactionBackend for TransactionServiceSqliteDatabase { Ok(()) } - fn confirm_broadcast_or_coinbase_transaction(&self, tx_id: u64) -> Result<(), TransactionStorageError> { + fn confirm_broadcast_or_coinbase_transaction(&self, _tx_id: u64) -> Result<(), TransactionStorageError> { + unimplemented!("obsolete"); + // let conn = self.database_connection.acquire_lock(); + // match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { + // Ok(v) => { + // if v.status == TransactionStatus::MinedUnconfirmed as i32 || + // v.status == TransactionStatus::MinedConfirmed as i32 || + // v.status == TransactionStatus::Broadcast as i32 || + // v.status == TransactionStatus::Coinbase as i32 + // { + // v.confirm(&(*conn))?; + // } else { + // return Err(TransactionStorageError::TransactionNotMined(tx_id)); + // } + // }, + // Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { + // return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( + // tx_id, + // ))); + // }, + // Err(e) => return Err(e), + // }; + // Ok(()) + } + + // fn set_completed_transaction_validity(&self, tx_id: u64, valid: bool) -> Result<(), TransactionStorageError> { + // let conn = self.database_connection.acquire_lock(); + // match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { + // Ok(v) => { + // v.set_validity(valid, &(*conn))?; + // }, + // Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { + // return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( + // tx_id, + // ))); + // }, + // Err(e) => return Err(e), + // }; + // Ok(()) + // } + + fn update_confirmations(&self, _tx_id: u64, _confirmations: u64) -> Result<(), TransactionStorageError> { + unimplemented!("obsolete"); + // let conn = self.database_connection.acquire_lock(); + // match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { + // Ok(v) => { + // v.update_confirmations(confirmations, &(*conn))?; + // }, + // Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { + // return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( + // tx_id, + // ))); + // }, + // Err(e) => return Err(e), + // }; + // Ok(()) + } + + fn update_mined_height( + &self, + tx_id: u64, + is_valid: bool, + mined_height: u64, + mined_in_block: BlockHash, + num_confirmations: u64, + is_confirmed: bool, + ) -> Result<(), TransactionStorageError> { let conn = self.database_connection.acquire_lock(); - match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { + match CompletedTransactionSql::find(tx_id, &(*conn)) { Ok(v) => { - if v.status == TransactionStatus::MinedUnconfirmed as i32 || - v.status == TransactionStatus::MinedConfirmed as i32 || - v.status == TransactionStatus::Broadcast as i32 || - v.status == TransactionStatus::Coinbase as i32 - { - v.confirm(&(*conn))?; - } else { - return Err(TransactionStorageError::TransactionNotMined(tx_id)); - } + v.update_mined_height( + is_valid, + mined_height, + mined_in_block, + num_confirmations, + is_confirmed, + &(*conn), + )?; }, Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( @@ -840,65 +865,47 @@ impl TransactionBackend for TransactionServiceSqliteDatabase { Ok(()) } - fn unconfirm_mined_transaction(&self, tx_id: u64) -> Result<(), TransactionStorageError> { + fn fetch_last_mined_transaction(&self) -> Result, TransactionStorageError> { let conn = self.database_connection.acquire_lock(); - match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { - Ok(v) => { - if v.status == TransactionStatus::MinedUnconfirmed as i32 || - v.status == TransactionStatus::MinedConfirmed as i32 - { - v.unconfirm(&(*conn))?; - } else { - return Err(TransactionStorageError::TransactionNotMined(tx_id)); - } + let tx = completed_transactions::table + .filter(completed_transactions::mined_height.is_not_null()) + .order_by(completed_transactions::mined_height.desc()) + .first::(&*conn) + .optional()?; + Ok(match tx { + Some(mut tx) => { + self.decrypt_if_necessary(&mut tx)?; + Some(tx.try_into()?) }, - Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { - return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( - tx_id, - ))); - }, - Err(e) => return Err(e), - }; - Ok(()) + None => None, + }) } - fn set_completed_transaction_validity(&self, tx_id: u64, valid: bool) -> Result<(), TransactionStorageError> { + fn fetch_unmined_transactions(&self) -> Result, TransactionStorageError> { let conn = self.database_connection.acquire_lock(); - match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { - Ok(v) => { - v.set_validity(valid, &(*conn))?; - }, - Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { - return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( - tx_id, - ))); - }, - Err(e) => return Err(e), - }; - Ok(()) - } + let txs = completed_transactions::table + .filter( + completed_transactions::mined_height + .is_null() + .or(completed_transactions::status.eq(TransactionStatus::MinedUnconfirmed as i32)), + ) + .order_by(completed_transactions::tx_id) + .load::(&*conn)?; - fn update_confirmations(&self, tx_id: u64, confirmations: u64) -> Result<(), TransactionStorageError> { - let conn = self.database_connection.acquire_lock(); - match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { - Ok(v) => { - v.update_confirmations(confirmations, &(*conn))?; - }, - Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { - return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( - tx_id, - ))); - }, - Err(e) => return Err(e), - }; - Ok(()) + let mut result = vec![]; + for mut tx in txs { + self.decrypt_if_necessary(&mut tx)?; + result.push(tx.try_into()?); + } + + Ok(result) } - fn update_mined_height(&self, tx_id: u64, mined_height: u64) -> Result<(), TransactionStorageError> { + fn set_transaction_as_unmined(&self, tx_id: u64) -> Result<(), TransactionStorageError> { let conn = self.database_connection.acquire_lock(); - match CompletedTransactionSql::find_by_cancelled(tx_id, false, &(*conn)) { + match CompletedTransactionSql::find(tx_id, &(*conn)) { Ok(v) => { - v.update_mined_height(mined_height, &(*conn))?; + v.set_as_unmined(&(*conn))?; }, Err(TransactionStorageError::DieselError(DieselError::NotFound)) => { return Err(TransactionStorageError::ValueNotFound(DbKey::CompletedTransaction( @@ -988,7 +995,7 @@ impl InboundTransactionSql { if num_updated == 0 { return Err(TransactionStorageError::UnexpectedResult( - "Database update error".to_string(), + "Updating inbound transactions failed. No rows were affected".to_string(), )); } @@ -1148,14 +1155,9 @@ impl OutboundTransactionSql { } pub fn delete(&self, conn: &SqliteConnection) -> Result<(), TransactionStorageError> { - let num_deleted = - diesel::delete(outbound_transactions::table.filter(outbound_transactions::tx_id.eq(&self.tx_id))) - .execute(conn)?; - - if num_deleted == 0 { - return Err(TransactionStorageError::ValuesNotFound); - } - + diesel::delete(outbound_transactions::table.filter(outbound_transactions::tx_id.eq(&self.tx_id))) + .execute(conn) + .num_rows_affected_or_not_found(1)?; Ok(()) } @@ -1164,16 +1166,10 @@ impl OutboundTransactionSql { update: UpdateOutboundTransactionSql, conn: &SqliteConnection, ) -> Result<(), TransactionStorageError> { - let num_updated = - diesel::update(outbound_transactions::table.filter(outbound_transactions::tx_id.eq(&self.tx_id))) - .set(update) - .execute(conn)?; - - if num_updated == 0 { - return Err(TransactionStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } + diesel::update(outbound_transactions::table.filter(outbound_transactions::tx_id.eq(&self.tx_id))) + .set(update) + .execute(conn) + .num_rows_affected_or_not_found(1)?; Ok(()) } @@ -1298,6 +1294,7 @@ struct CompletedTransactionSql { valid: i32, confirmations: Option, mined_height: Option, + mined_in_block: Option>, } impl CompletedTransactionSql { @@ -1365,33 +1362,18 @@ impl CompletedTransactionSql { updated_tx: UpdateCompletedTransactionSql, conn: &SqliteConnection, ) -> Result<(), TransactionStorageError> { - let num_updated = - diesel::update(completed_transactions::table.filter(completed_transactions::tx_id.eq(&self.tx_id))) - .set(updated_tx) - .execute(conn)?; - - if num_updated == 0 { - return Err(TransactionStorageError::UnexpectedResult( - "Database update error".to_string(), - )); - } - + diesel::update(completed_transactions::table.filter(completed_transactions::tx_id.eq(&self.tx_id))) + .set(updated_tx) + .execute(conn) + .num_rows_affected_or_not_found(1)?; Ok(()) } pub fn cancel(&self, conn: &SqliteConnection) -> Result<(), TransactionStorageError> { self.update( UpdateCompletedTransactionSql { - status: None, - timestamp: None, cancelled: Some(1i32), - direction: None, - transaction_protocol: None, - send_count: None, - last_send_timestamp: None, - valid: None, - confirmations: None, - mined_height: None, + ..Default::default() }, conn, )?; @@ -1399,62 +1381,23 @@ impl CompletedTransactionSql { Ok(()) } - pub fn confirm(&self, conn: &SqliteConnection) -> Result<(), TransactionStorageError> { + pub fn set_as_unmined(&self, conn: &SqliteConnection) -> Result<(), TransactionStorageError> { self.update( UpdateCompletedTransactionSql { - status: Some(TransactionStatus::MinedConfirmed as i32), - timestamp: None, - cancelled: None, - direction: None, - transaction_protocol: None, - send_count: None, - last_send_timestamp: None, - valid: None, - confirmations: None, - mined_height: None, + // TODO: Coinbases should technically go back to 'Coinbase' instead of 'Completed' + status: Some(TransactionStatus::Completed as i32), + mined_in_block: Some(None), + mined_height: Some(None), + confirmations: Some(None), + // Resets to valid + valid: Some(1), + ..Default::default() }, conn, )?; - Ok(()) - } - - pub fn unconfirm(&self, conn: &SqliteConnection) -> Result<(), TransactionStorageError> { - self.update( - UpdateCompletedTransactionSql { - status: Some(TransactionStatus::MinedUnconfirmed as i32), - timestamp: None, - cancelled: None, - direction: None, - transaction_protocol: None, - send_count: None, - last_send_timestamp: None, - valid: None, - confirmations: None, - mined_height: None, - }, - conn, - )?; - - Ok(()) - } - - pub fn set_validity(&self, valid: bool, conn: &SqliteConnection) -> Result<(), TransactionStorageError> { - self.update( - UpdateCompletedTransactionSql { - status: None, - timestamp: None, - cancelled: None, - direction: None, - transaction_protocol: None, - send_count: None, - last_send_timestamp: None, - valid: Some(valid as i32), - confirmations: None, - mined_height: None, - }, - conn, - )?; + // Ideally the outputs should be marked unmined here as well, but because of the separation of classes, + // that will be done in the outputs service. Ok(()) } @@ -1462,40 +1405,8 @@ impl CompletedTransactionSql { pub fn update_encryption(&self, conn: &SqliteConnection) -> Result<(), TransactionStorageError> { self.update( UpdateCompletedTransactionSql { - status: None, - timestamp: None, - cancelled: None, - direction: None, transaction_protocol: Some(self.transaction_protocol.clone()), - send_count: None, - last_send_timestamp: None, - valid: None, - confirmations: None, - mined_height: None, - }, - conn, - )?; - - Ok(()) - } - - pub fn update_confirmations( - &self, - confirmations: u64, - conn: &SqliteConnection, - ) -> Result<(), TransactionStorageError> { - self.update( - UpdateCompletedTransactionSql { - status: None, - timestamp: None, - cancelled: None, - direction: None, - transaction_protocol: Some(self.transaction_protocol.clone()), - send_count: None, - last_send_timestamp: None, - valid: None, - confirmations: Some(Some(confirmations as i64)), - mined_height: None, + ..Default::default() }, conn, )?; @@ -1505,21 +1416,27 @@ impl CompletedTransactionSql { pub fn update_mined_height( &self, + is_valid: bool, mined_height: u64, + mined_in_block: BlockHash, + num_confirmations: u64, + is_confirmed: bool, conn: &SqliteConnection, ) -> Result<(), TransactionStorageError> { self.update( UpdateCompletedTransactionSql { - status: None, - timestamp: None, - cancelled: None, - direction: None, - transaction_protocol: None, - send_count: None, - last_send_timestamp: None, - valid: None, - confirmations: None, + confirmations: Some(Some(num_confirmations as i64)), + status: Some(if is_confirmed { + TransactionStatus::MinedConfirmed as i32 + } else { + TransactionStatus::MinedUnconfirmed as i32 + }), mined_height: Some(Some(mined_height as i64)), + mined_in_block: Some(Some(mined_in_block)), + valid: Some(if is_valid { 1 } else { 0 }), + // If the tx is mined, then it can't be cancelled + cancelled: Some(0), + ..Default::default() }, conn, )?; @@ -1570,6 +1487,7 @@ impl TryFrom for CompletedTransactionSql { valid: c.valid as i32, confirmations: c.confirmations.map(|ic| ic as i64), mined_height: c.mined_height.map(|ic| ic as i64), + mined_in_block: c.mined_in_block, }) } } @@ -1598,24 +1516,12 @@ impl TryFrom for CompletedTransaction { valid: c.valid != 0, confirmations: c.confirmations.map(|ic| ic as u64), mined_height: c.mined_height.map(|ic| ic as u64), + mined_in_block: c.mined_in_block, }) } } -/// These are the fields that can be updated for a Completed Transaction -pub struct UpdateCompletedTransaction { - status: Option, - timestamp: Option, - cancelled: Option, - direction: Option, - send_count: Option, - last_send_timestamp: Option>, - valid: Option, - confirmations: Option>, - mined_height: Option>, -} - -#[derive(AsChangeset)] +#[derive(AsChangeset, Default)] #[table_name = "completed_transactions"] pub struct UpdateCompletedTransactionSql { status: Option, @@ -1628,24 +1534,7 @@ pub struct UpdateCompletedTransactionSql { valid: Option, confirmations: Option>, mined_height: Option>, -} - -/// Map a Rust friendly UpdateCompletedTransaction to the Sql data type form -impl From for UpdateCompletedTransactionSql { - fn from(u: UpdateCompletedTransaction) -> Self { - Self { - status: u.status.map(|s| s as i32), - timestamp: u.timestamp, - cancelled: u.cancelled.map(|c| c as i32), - direction: u.direction.map(|d| d as i32), - transaction_protocol: None, - send_count: u.send_count.map(|c| c as i32), - last_send_timestamp: u.last_send_timestamp, - valid: u.valid.map(|c| c as i32), - confirmations: u.confirmations.map(|c| c.map(|ic| ic as i64)), - mined_height: u.mined_height.map(|c| c.map(|ic| ic as i64)), - } - } + mined_in_block: Option>>, } #[cfg(test)] diff --git a/base_layer/wallet/src/util/diesel_ext.rs b/base_layer/wallet/src/util/diesel_ext.rs new file mode 100644 index 0000000000..3af2cd67aa --- /dev/null +++ b/base_layer/wallet/src/util/diesel_ext.rs @@ -0,0 +1,42 @@ +// Copyright 2021. The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use diesel::{result::Error as DieselError, QueryResult}; + +pub trait ExpectedRowsExtension { + fn num_rows_affected_or_not_found(self, num_rows: usize) -> Result; +} + +impl ExpectedRowsExtension for QueryResult { + fn num_rows_affected_or_not_found(self, num_rows: usize) -> Result { + match self { + Ok(s) => { + if s == num_rows { + Ok(s) + } else { + Err(DieselError::NotFound) + } + }, + Err(e) => Err(e), + } + } +} diff --git a/base_layer/wallet/src/util/mod.rs b/base_layer/wallet/src/util/mod.rs index 9664a0e376..5c800244a8 100644 --- a/base_layer/wallet/src/util/mod.rs +++ b/base_layer/wallet/src/util/mod.rs @@ -20,6 +20,7 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +pub mod diesel_ext; pub mod emoji; pub mod encryption; pub mod luhn; diff --git a/base_layer/wallet/src/utxo_scanner_service/utxo_scanning.rs b/base_layer/wallet/src/utxo_scanner_service/utxo_scanning.rs index 6aaa38363f..5077be6e6c 100644 --- a/base_layer/wallet/src/utxo_scanner_service/utxo_scanning.rs +++ b/base_layer/wallet/src/utxo_scanner_service/utxo_scanning.rs @@ -306,6 +306,7 @@ where TBackend: WalletBackend + 'static return Ok((total_scanned, start_index, timer.elapsed())); } + // TODO: uncomment let num_scanned = self.scan_utxos(&mut client, start_index, tip_header).await?; if num_scanned == 0 { return Err(UtxoScannerError::UtxoScanningError( @@ -319,7 +320,10 @@ where TBackend: WalletBackend + 'static timer.elapsed(), num_scanned ); + + // let num_scanned = 0; total_scanned += num_scanned; + // return Ok((total_scanned, start_index, timer.elapsed())); } } diff --git a/comms/src/peer_manager/node_id.rs b/comms/src/peer_manager/node_id.rs index e6cc0c1259..5d6dc23387 100644 --- a/comms/src/peer_manager/node_id.rs +++ b/comms/src/peer_manager/node_id.rs @@ -357,6 +357,12 @@ impl TryFrom<&[u8]> for NodeId { } } +impl From for NodeId { + fn from(pk: CommsPublicKey) -> Self { + NodeId::from_public_key(&pk) + } +} + impl Hash for NodeId { /// Require the implementation of the Hash trait for Hashmaps fn hash(&self, state: &mut H) { diff --git a/comms/src/protocol/rpc/handshake.rs b/comms/src/protocol/rpc/handshake.rs index 7c4dca1ae9..f2141fa478 100644 --- a/comms/src/protocol/rpc/handshake.rs +++ b/comms/src/protocol/rpc/handshake.rs @@ -132,9 +132,18 @@ where T: AsyncRead + AsyncWrite + Unpin debug!(target: LOG_TARGET, "Server accepted version {}", version); Ok(()) }, - Ok(Some(Err(err))) => Err(err.into()), + Ok(Some(Err(err))) => { + error!(target: LOG_TARGET, "RPC client handshake error: {}", err); + Err(err.into()) + }, Ok(None) => Err(RpcHandshakeError::ServerClosedRequest), - Err(_) => Err(RpcHandshakeError::TimedOut), + Err(err) => { + error!( + target: LOG_TARGET, + "RPC client handshake error(probably a timeout): {}", err + ); + Err(RpcHandshakeError::TimedOut) + }, } } diff --git a/comms/src/protocol/rpc/server/mod.rs b/comms/src/protocol/rpc/server/mod.rs index 7615b1497e..6591400ce7 100644 --- a/comms/src/protocol/rpc/server/mod.rs +++ b/comms/src/protocol/rpc/server/mod.rs @@ -56,6 +56,7 @@ use crate::{ use futures::{channel::mpsc, AsyncRead, AsyncWrite, SinkExt, StreamExt}; use log::*; use prost::Message; +use rand::{rngs::OsRng, RngCore}; use std::{ borrow::Cow, future::Future, @@ -356,6 +357,7 @@ where ); let service = ActivePeerRpcService { + id: OsRng.next_u64(), config: self.config.clone(), protocol, node_id: node_id.clone(), @@ -373,6 +375,7 @@ where } struct ActivePeerRpcService { + id: u64, config: RpcServerBuilder, protocol: ProtocolId, node_id: NodeId, diff --git a/integration_tests/features/WalletBaseNodeSwitch.feature b/integration_tests/features/WalletBaseNodeSwitch.feature new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration_tests/features/WalletMonitoring.feature b/integration_tests/features/WalletMonitoring.feature index 09875c8306..a3588c3282 100644 --- a/integration_tests/features/WalletMonitoring.feature +++ b/integration_tests/features/WalletMonitoring.feature @@ -1,4 +1,4 @@ -@coinbase_reorg +@coinbase_reorg @wallet Feature: Wallet Monitoring Scenario: Wallets monitoring coinbase after a reorg diff --git a/integration_tests/features/WalletPasswordChange.feature b/integration_tests/features/WalletPasswordChange.feature new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration_tests/features/WalletQuery.feature b/integration_tests/features/WalletQuery.feature index 825854a312..7c022bdc52 100644 --- a/integration_tests/features/WalletQuery.feature +++ b/integration_tests/features/WalletQuery.feature @@ -1,3 +1,4 @@ +@wallet Feature: Wallet Querying Scenario: As a wallet I want to query the status of utxos in blocks diff --git a/integration_tests/features/WalletRecovery.feature b/integration_tests/features/WalletRecovery.feature index f52be08cce..1e38b208d2 100644 --- a/integration_tests/features/WalletRecovery.feature +++ b/integration_tests/features/WalletRecovery.feature @@ -1,4 +1,4 @@ -@wallet-recovery +@wallet-recovery @wallet Feature: Wallet Recovery @critical @@ -48,4 +48,4 @@ Feature: Wallet Recovery Then I wait for wallet WALLET_C to have less than 100000 uT When I merge mine 5 blocks via PROXY Then all nodes are at height 25 - Then I wait for wallet WALLET_C to have at least 1000000 uT \ No newline at end of file + Then I wait for wallet WALLET_C to have at least 1000000 uT diff --git a/integration_tests/features/WalletRoutingMechanism.feature b/integration_tests/features/WalletRoutingMechanism.feature index 7363c5d5ee..e1f4a9d87e 100644 --- a/integration_tests/features/WalletRoutingMechanism.feature +++ b/integration_tests/features/WalletRoutingMechanism.feature @@ -1,4 +1,4 @@ -@routing_mechanism +@routing_mechanism @wallet Feature: Wallet Routing Mechanism Scenario Outline: Wallets transacting via specified routing mechanism only