feat!: new transaction and output validation protocol (#3421)

alternate approach to transaction validation (#3191)
feat!: integration of new transaction and output validation (#3352)
feat: update coinbase handling for new tx and output validation (#3383)
refactor: use wallet connectivity in wallet services (#3391)
refactor: mapping for deleted mmr position to height/hash for perf (#3394)
feat: Fix reinstating CancelledInbound outputs and fix tests (#3400)
chore: merge development into tx-validation2 and fix issues (#3417)
Uncomment cucumber tests

applications/daily_tests/cron_jobs.js

@@ -52,10 +52,10 @@ async function runWalletRecoveryTest(instances) {
       scannedRate,
       recoveredAmount,
     } = await walletRecoveryTest({
-      seedWords:
-        "spare man patrol essay divide hollow trip visual actress sadness country hungry toy blouse body club depend capital sleep aim high recycle crystal abandon",
-      log: LOG_FILE,
-      numWallets: instances,
+        seedWords:
+          "spare man patrol essay divide hollow trip visual actress sadness country hungry toy blouse body club depend capital sleep aim high recycle crystal abandon",
+        log: LOG_FILE,
+        numWallets: instances,
       baseDir,
     });
 

applications/ffi_client/index.js

@@ -195,9 +195,9 @@ try {
       let id = lib.wallet_start_transaction_validation(wallet, err);
       console.log("tx validation request id", id);
 
-      console.log("start utxo validation");
-      id = lib.wallet_start_utxo_validation(wallet, err);
-      console.log("utxo validation request id", id);
+      console.log("start txo validation");
+      id = lib.wallet_start_txo_validation(wallet, err);
+      console.log("txo validation request id", id);
     } catch (e) {
       console.error("validation error: ", e);
     }

applications/ffi_client/lib/index.js

@@ -74,7 +74,7 @@ const libWallet = ffi.Library("./libtari_wallet_ffi.dylib", {
   wallet_get_num_confirmations_required: [u64, [walletRef, errPtr]],
   wallet_set_num_confirmations_required: ["void", [walletRef, u64, errPtr]],
   wallet_start_transaction_validation: [u64, [walletRef, errPtr]],
-  wallet_start_utxo_validation: [u64, [walletRef, errPtr]],
+  wallet_start_txo_validation: [u64, [walletRef, errPtr]],
   wallet_start_recovery: [bool, [walletRef, publicKeyRef, fn, errPtr]],
 });
 

applications/tari_app_utilities/src/utilities.rs

@@ -78,6 +78,8 @@ pub enum ExitCodes {
     NoPassword,
     #[error("Tor connection is offline")]
     TorOffline,
+    #[error("Database is in inconsistent state: {0}")]
+    DbInconsistentState(String),
 }
 
 impl ExitCodes {
@@ -96,6 +98,29 @@ impl ExitCodes {
             Self::ConversionError(_) => 111,
             Self::IncorrectPassword | Self::NoPassword => 112,
             Self::TorOffline => 113,
+            Self::DbInconsistentState(_) => 115,
+        }
+    }
+
+    pub fn eprint_details(&self) {
+        use ExitCodes::*;
+        match self {
+            TorOffline => {
+                eprintln!("Unable to connect to the Tor control port.");
+                eprintln!(
+                    "Please check that you have the Tor proxy running and that access to the Tor control port is \
+                     turned on.",
+                );
+                eprintln!("If you are unsure of what to do, use the following command to start the Tor proxy:");
+                eprintln!(
+                    "tor --allow-missing-torrc --ignore-missing-torrc --clientonly 1 --socksport 9050 --controlport \
+                     127.0.0.1:9051 --log \"notice stdout\" --clientuseipv6 1",
+                );
+            },
+
+            e => {
+                eprintln!("{}", e);
+            },
         }
     }
 }

applications/tari_base_node/src/main.rs

@@ -115,6 +115,7 @@ use tari_app_utilities::{
 };
 use tari_common::{configuration::bootstrap::ApplicationType, ConfigBootstrap, GlobalConfig};
 use tari_comms::{peer_manager::PeerFeatures, tor::HiddenServiceControllerError};
+use tari_core::chain_storage::ChainStorageError;
 use tari_shutdown::{Shutdown, ShutdownSignal};
 use tokio::{
     runtime,
@@ -128,7 +129,7 @@ const LOG_TARGET: &str = "base_node::app";
 /// Application entry point
 fn main() {
     if let Err(exit_code) = main_inner() {
-        eprintln!("{:?}", exit_code);
+        exit_code.eprint_details();
         error!(
             target: LOG_TARGET,
             "Exiting with code ({}): {:?}",
@@ -205,21 +206,15 @@ async fn run_node(node_config: Arc<GlobalConfig>, bootstrap: ConfigBootstrap) ->
     .await
     .map_err(|err| {
         for boxed_error in err.chain() {
-            if let Some(HiddenServiceControllerError::TorControlPortOffline) =
-                boxed_error.downcast_ref::<HiddenServiceControllerError>()
-            {
-                println!("Unable to connect to the Tor control port.");
-                println!(
-                    "Please check that you have the Tor proxy running and that access to the Tor control port is \
-                     turned on.",
-                );
-                println!("If you are unsure of what to do, use the following command to start the Tor proxy:");
-                println!(
-                    "tor --allow-missing-torrc --ignore-missing-torrc --clientonly 1 --socksport 9050 --controlport \
-                     127.0.0.1:9051 --log \"notice stdout\" --clientuseipv6 1",
-                );
+            if let Some(HiddenServiceControllerError::TorControlPortOffline) = boxed_error.downcast_ref() {
                 return ExitCodes::TorOffline;
             }
+            if let Some(ChainStorageError::DatabaseResyncRequired(reason)) = boxed_error.downcast_ref() {
+                return ExitCodes::DbInconsistentState(format!(
+                    "You may need to resync your database because {}",
+                    reason
+                ));
+            }
 
             // todo: find a better way to do this
             if boxed_error.to_string().contains("Invalid force sync peer") {

applications/tari_console_wallet/Cargo.toml

@@ -56,4 +56,4 @@ default-features = false
 features = ["crossterm"]
 
 [features]
-avx2 = []
+avx2 = []

applications/tari_console_wallet/src/automation/commands.rs

@@ -500,26 +500,36 @@ pub async fn monitor_transactions(
                         }
                     }
                 },
-                TransactionEvent::TransactionMinedUnconfirmed(id, confirmations) if tx_ids.contains(id) => {
+                TransactionEvent::TransactionMinedUnconfirmed {
+                    tx_id,
+                    num_confirmations,
+                    is_valid,
+                } if tx_ids.contains(tx_id) => {
                     debug!(
                         target: LOG_TARGET,
-                        "tx mined unconfirmed event for tx_id: {}, confirmations: {}", *id, confirmations
+                        "tx mined unconfirmed event for tx_id: {}, confirmations: {}, is_valid: {}",
+                        *tx_id,
+                        num_confirmations,
+                        is_valid
                     );
                     if wait_stage == TransactionStage::MinedUnconfirmed {
                         results.push(SentTransaction {
-                            id: *id,
+                            id: *tx_id,
                             stage: TransactionStage::MinedUnconfirmed,
                         });
                         if results.len() == tx_ids.len() {
                             break;
                         }
                     }
                 },
-                TransactionEvent::TransactionMined(id) if tx_ids.contains(id) => {
-                    debug!(target: LOG_TARGET, "tx mined confirmed event for tx_id: {}", *id);
+                TransactionEvent::TransactionMined { tx_id, is_valid } if tx_ids.contains(tx_id) => {
+                    debug!(
+                        target: LOG_TARGET,
+                        "tx mined confirmed event for tx_id: {}, is_valid:{}", *tx_id, is_valid
+                    );
                     if wait_stage == TransactionStage::Mined {
                         results.push(SentTransaction {
-                            id: *id,
+                            id: *tx_id,
                             stage: TransactionStage::Mined,
                         });
                         if results.len() == tx_ids.len() {

applications/tari_console_wallet/src/init/mod.rs

@@ -47,13 +47,9 @@ use tari_shutdown::ShutdownSignal;
 use tari_wallet::{
     base_node_service::config::BaseNodeServiceConfig,
     error::{WalletError, WalletStorageError},
-    output_manager_service::{config::OutputManagerServiceConfig, TxoValidationType},
+    output_manager_service::config::OutputManagerServiceConfig,
     storage::{database::WalletDatabase, sqlite_utilities::initialize_sqlite_database_backends},
-    transaction_service::{
-        config::{TransactionRoutingMechanism, TransactionServiceConfig},
-        tasks::start_transaction_validation_and_broadcast_protocols::start_transaction_validation_and_broadcast_protocols,
-    },
-    types::ValidationRetryStrategy,
+    transaction_service::config::{TransactionRoutingMechanism, TransactionServiceConfig},
     Wallet,
     WalletConfig,
     WalletSqlite,
@@ -390,7 +386,7 @@ pub async fn init_wallet(
             base_node_query_timeout: config.base_node_query_timeout,
             prevent_fee_gt_amount: config.prevent_fee_gt_amount,
             event_channel_size: config.output_manager_event_channel_size,
-            base_node_update_publisher_channel_size: config.base_node_update_publisher_channel_size,
+            num_confirmations_required: config.transaction_num_confirmations_required,
             ..Default::default()
         }),
         config.network.into(),
@@ -500,12 +496,7 @@ pub async fn start_wallet(
         if let Err(e) = wallet.transaction_service.restart_transaction_protocols().await {
             error!(target: LOG_TARGET, "Problem restarting transaction protocols: {}", e);
         }
-        if let Err(e) = start_transaction_validation_and_broadcast_protocols(
-            wallet.transaction_service.clone(),
-            ValidationRetryStrategy::UntilSuccess,
-        )
-        .await
-        {
+        if let Err(e) = wallet.transaction_service.validate_transactions().await {
             error!(
                 target: LOG_TARGET,
                 "Problem validating and restarting transaction protocols: {}", e
@@ -521,37 +512,12 @@ pub async fn start_wallet(
 async fn validate_txos(wallet: &mut WalletSqlite) -> Result<(), ExitCodes> {
     debug!(target: LOG_TARGET, "Starting TXO validations.");
 
-    // Unspent TXOs
-    wallet
-        .output_manager_service
-        .validate_txos(TxoValidationType::Unspent, ValidationRetryStrategy::UntilSuccess)
-        .await
-        .map_err(|e| {
-            error!(target: LOG_TARGET, "Error validating Unspent TXOs: {}", e);
-            ExitCodes::WalletError(e.to_string())
-        })?;
-
-    // Spent TXOs
-    wallet
-        .output_manager_service
-        .validate_txos(TxoValidationType::Spent, ValidationRetryStrategy::UntilSuccess)
-        .await
-        .map_err(|e| {
-            error!(target: LOG_TARGET, "Error validating Spent TXOs: {}", e);
-            ExitCodes::WalletError(e.to_string())
-        })?;
-
-    // Invalid TXOs
-    wallet
-        .output_manager_service
-        .validate_txos(TxoValidationType::Invalid, ValidationRetryStrategy::UntilSuccess)
-        .await
-        .map_err(|e| {
-            error!(target: LOG_TARGET, "Error validating Invalid TXOs: {}", e);
-            ExitCodes::WalletError(e.to_string())
-        })?;
+    wallet.output_manager_service.validate_txos().await.map_err(|e| {
+        error!(target: LOG_TARGET, "Error validating Unspent TXOs: {}", e);
+        ExitCodes::WalletError(e.to_string())
+    })?;
 
-    debug!(target: LOG_TARGET, "TXO validations completed.");
+    debug!(target: LOG_TARGET, "TXO validations started.");
 
     Ok(())
 }

applications/tari_console_wallet/src/ui/components/base_node.rs

@@ -21,7 +21,7 @@
 // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 use crate::ui::{components::Component, state::AppState};
-use tari_wallet::connectivity_service::OnlineStatus;
+use tari_wallet::connectivity_service::{OnlineStatus, WalletConnectivityInterface};
 use tui::{
     backend::Backend,
     layout::Rect,

applications/tari_console_wallet/src/ui/components/transactions_tab.rs

@@ -208,7 +208,7 @@ impl TransactionsTab {
                 format!("{}", local_time.format("%Y-%m-%d %H:%M:%S")),
                 Style::default().fg(text_color),
             )));
-            let status = if t.cancelled && t.status == TransactionStatus::Coinbase {
+            let status = if (t.cancelled || !t.valid) && t.status == TransactionStatus::Coinbase {
                 "Abandoned".to_string()
             } else if t.cancelled {
                 "Cancelled".to_string()

applications/tari_console_wallet/src/ui/state/app_state.rs

@@ -52,12 +52,11 @@ use tari_wallet::{
     base_node_service::{handle::BaseNodeEventReceiver, service::BaseNodeState},
     connectivity_service::WalletConnectivityHandle,
     contacts_service::storage::database::Contact,
-    output_manager_service::{handle::OutputManagerEventReceiver, service::Balance, TxId, TxoValidationType},
+    output_manager_service::{handle::OutputManagerEventReceiver, service::Balance, TxId},
     transaction_service::{
         handle::TransactionEventReceiver,
         storage::models::{CompletedTransaction, TransactionStatus},
     },
-    types::ValidationRetryStrategy,
     WalletSqlite,
 };
 
@@ -351,7 +350,7 @@ impl AppState {
             self.cached_data
                 .completed_txs
                 .iter()
-                .filter(|tx| !(tx.cancelled && tx.status == TransactionStatus::Coinbase))
+                .filter(|tx| !((tx.cancelled || !tx.valid) && tx.status == TransactionStatus::Coinbase))
                 .collect()
         } else {
             self.cached_data.completed_txs.iter().collect()
@@ -815,33 +814,13 @@ impl AppStateInner {
         let mut output_manager_service = self.wallet.output_manager_service.clone();
 
         task::spawn(async move {
-            if let Err(e) = txn_service
-                .validate_transactions(ValidationRetryStrategy::UntilSuccess)
-                .await
-            {
+            if let Err(e) = txn_service.validate_transactions().await {
                 error!(target: LOG_TARGET, "Problem validating transactions: {}", e);
             }
 
-            if let Err(e) = output_manager_service
-                .validate_txos(TxoValidationType::Unspent, ValidationRetryStrategy::UntilSuccess)
-                .await
-            {
+            if let Err(e) = output_manager_service.validate_txos().await {
                 error!(target: LOG_TARGET, "Problem validating UTXOs: {}", e);
             }
-
-            if let Err(e) = output_manager_service
-                .validate_txos(TxoValidationType::Spent, ValidationRetryStrategy::UntilSuccess)
-                .await
-            {
-                error!(target: LOG_TARGET, "Problem validating STXOs: {}", e);
-            }
-
-            if let Err(e) = output_manager_service
-                .validate_txos(TxoValidationType::Invalid, ValidationRetryStrategy::UntilSuccess)
-                .await
-            {
-                error!(target: LOG_TARGET, "Problem validating Invalid TXOs: {}", e);
-            }
         });
     }