Merge branch 'development' into merge-dev-feature-dan

* development:
  fix: add hidden types and seed words to key manager (tari-project#4925)
  feat: timestamp validation (tari-project#4887)
  fix: deleted_txo_mmr_position_to_height_index  already exists error (tari-project#4924)
  feat: add default grpc for localnet (tari-project#4937)
  first commit (tari-project#4926)
  v0.40.2
  fix(dht): use limited ban period for invalid peer (tari-project#4933)
  feat: upgrade tari_crypto sign api (tari-project#4932)
  v0.40.1
  chore: fix depreciated timestamp clippy (tari-project#4929)
  chore: fix naming of ffi functions and comments  (tari-project#4930)
  fix: set wallet start scan height to birthday and not 0 (see issue tari-project#4807) (tari-project#4911)
  v0.40.0
  chore: remove unused methods (tari-project#4922)
  fix: updates for SafePassword API change (tari-project#4927)

applications/tari_app_grpc/Cargo.toml

@@ -4,16 +4,16 @@ authors = ["The Tari Development Community"]
 description = "This crate is to provide a single source for all cross application grpc files and conversions to and from tari::core"
 repository = "https://github.com/tari-project/tari"
 license = "BSD-3-Clause"
-version = "0.39.0"
+version = "0.40.2"
 edition = "2018"
 
 [dependencies]
-tari_common_types = { version = "^0.39", path = "../../base_layer/common_types" }
+tari_common_types = { version = "^0.40", path = "../../base_layer/common_types" }
 tari_comms = { path = "../../comms/core" }
 tari_core = { path = "../../base_layer/core" }
-tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.15.7" }
+tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.16.2" }
 tari_script = { path = "../../infrastructure/tari_script" }
-tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.7" }
+tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.9" }
 
 argon2 = { version = "0.4.1", features = ["std", "password-hash"] }
 base64 = "0.13.0"

applications/tari_app_utilities/Cargo.toml

@@ -1,16 +1,16 @@
 [package]
 name = "tari_app_utilities"
-version = "0.39.0"
+version = "0.40.2"
 authors = ["The Tari Development Community"]
 edition = "2018"
 license = "BSD-3-Clause"
 
 [dependencies]
 tari_comms = { path = "../../comms/core" }
-tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.15.7" }
+tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.16.2" }
 tari_common = { path = "../../common" }
 tari_common_types = { path = "../../base_layer/common_types" }
-tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.7" }
+tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.9" }
 
 clap = { version = "3.2.0", features = ["derive", "env"] }
 config = { version = "0.13.0" }

applications/tari_base_node/Cargo.toml

@@ -4,7 +4,7 @@ authors = ["The Tari Development Community"]
 description = "The tari full base node implementation"
 repository = "https://github.com/tari-project/tari"
 license = "BSD-3-Clause"
-version = "0.39.0"
+version = "0.40.2"
 edition = "2018"
 
 [dependencies]
@@ -15,14 +15,14 @@ tari_comms = { path = "../../comms/core", features = ["rpc"] }
 tari_common_types = { path = "../../base_layer/common_types" }
 tari_comms_dht = { path = "../../comms/dht" }
 tari_core = { path = "../../base_layer/core", default-features = false, features = ["transactions"] }
-tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.15.7" }
+tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.16.2" }
 tari_libtor = { path = "../../infrastructure/libtor", optional = true }
 tari_mmr = { path = "../../base_layer/mmr", features = ["native_bitmap"] }
 tari_p2p = { path = "../../base_layer/p2p", features = ["auto-update"] }
 tari_storage = {path="../../infrastructure/storage"}
 tari_service_framework = { path = "../../base_layer/service_framework" }
 tari_shutdown = { path = "../../infrastructure/shutdown" }
-tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.7" }
+tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.9" }
 
 anyhow = "1.0.53"
 async-trait = "0.1.52"

applications/tari_console_wallet/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "tari_console_wallet"
-version = "0.39.0"
+version = "0.40.2"
 authors = ["The Tari Development Community"]
 edition = "2018"
 license = "BSD-3-Clause"
 
 [dependencies]
 tari_wallet = { path = "../../base_layer/wallet", features = ["bundled_sqlite"] }
-tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.15.7" }
+tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.16.2" }
 tari_common = { path = "../../common" }
 tari_app_utilities = { path = "../tari_app_utilities" }
 tari_comms = { path = "../../comms/core" }
@@ -18,7 +18,7 @@ tari_p2p = { path = "../../base_layer/p2p", features = ["auto-update"] }
 tari_app_grpc = { path = "../tari_app_grpc" }
 tari_shutdown = { path = "../../infrastructure/shutdown" }
 tari_key_manager = { path = "../../base_layer/key_manager" }
-tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.7" }
+tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.9" }
 
 # Uncomment for tokio tracing via tokio-console (needs "tracing" featurs)
 #console-subscriber = "0.1.3"
@@ -50,6 +50,7 @@ tonic = "0.6.2"
 tracing = "0.1.26"
 unicode-segmentation = "1.6.0"
 unicode-width = "0.1"
+zeroize = "1.3.0"
 
 [dependencies.tari_core]
 path = "../../base_layer/core"

applications/tari_console_wallet/src/init/mod.rs

@@ -401,7 +401,7 @@ pub async fn init_wallet(
     }
     if let Some(file_name) = seed_words_file_name {
         let seed_words = wallet.get_seed_words(&MnemonicLanguage::English)?.join(" ");
-        let _result = fs::write(file_name, seed_words).map_err(|e| {
+        let _result = fs::write(file_name, seed_words.reveal()).map_err(|e| {
             ExitError::new(
                 ExitCode::WalletError,
                 &format!("Problem writing seed words to file: {}", e),
@@ -549,7 +549,7 @@ fn confirm_seed_words(wallet: &mut WalletSqlite) -> Result<(), ExitError> {
     println!("WRITE THEM DOWN OR COPY THEM NOW. THIS IS YOUR ONLY CHANCE TO DO SO.");
     println!();
     println!("=========================");
-    println!("{}", seed_words.join(" "));
+    println!("{}", seed_words.join(" ").reveal());
     println!("=========================");
     println!("\x07"); // beep!
 

applications/tari_console_wallet/src/lib.rs

@@ -48,7 +48,8 @@ use tari_common::{
     configuration::bootstrap::ApplicationType,
     exit_codes::{ExitCode, ExitError},
 };
-use tari_key_manager::cipher_seed::CipherSeed;
+use tari_crypto::tari_utilities::Hidden;
+use tari_key_manager::{cipher_seed::CipherSeed, SeedWords};
 #[cfg(all(unix, feature = "libtor"))]
 use tari_libtor::tor::Tor;
 use tari_shutdown::Shutdown;
@@ -213,13 +214,15 @@ fn get_password(config: &ApplicationConfig, cli: &Cli) -> Option<SafePassword> {
 fn get_recovery_seed(boot_mode: WalletBoot, cli: &Cli) -> Result<Option<CipherSeed>, ExitError> {
     if matches!(boot_mode, WalletBoot::Recovery) {
         let seed = if cli.seed_words.is_some() {
-            let seed_words: Vec<String> = cli
-                .seed_words
-                .clone()
-                .unwrap()
-                .split_whitespace()
-                .map(|v| v.to_string())
-                .collect();
+            // need to zeroize first, to clean up memory of cli.seed_words clone
+            let seed_words: SeedWords = SeedWords::new(
+                cli.seed_words
+                    .as_ref()
+                    .unwrap()
+                    .split_whitespace()
+                    .map(|s| Hidden::hide(s.to_string()))
+                    .collect(),
+            );
             get_seed_from_seed_words(seed_words)?
         } else {
             prompt_private_key_from_seed_words()?

applications/tari_console_wallet/src/recovery.rs

@@ -27,7 +27,8 @@ use futures::FutureExt;
 use log::*;
 use rustyline::Editor;
 use tari_common::exit_codes::{ExitCode, ExitError};
-use tari_key_manager::{cipher_seed::CipherSeed, mnemonic::Mnemonic};
+use tari_crypto::tari_utilities::Hidden;
+use tari_key_manager::{cipher_seed::CipherSeed, mnemonic::Mnemonic, SeedWords};
 use tari_shutdown::Shutdown;
 use tari_utilities::hex::Hex;
 use tari_wallet::{
@@ -37,6 +38,7 @@ use tari_wallet::{
     WalletSqlite,
 };
 use tokio::sync::broadcast;
+use zeroize::Zeroizing;
 
 use crate::wallet_modes::PeerConfig;
 
@@ -51,8 +53,9 @@ pub fn prompt_private_key_from_seed_words() -> Result<CipherSeed, ExitError> {
         println!("Recovery Mode");
         println!();
         println!("Type or paste all of your seed words on one line, only separated by spaces.");
-        let input = rl.readline(">> ").map_err(|e| ExitError::new(ExitCode::IOError, e))?;
-        let seed_words: Vec<String> = input.split_whitespace().map(str::to_string).collect();
+        let input = Zeroizing::new(rl.readline(">> ").map_err(|e| ExitError::new(ExitCode::IOError, e))?);
+        let seed_words: SeedWords =
+            SeedWords::new(input.split_whitespace().map(|s| Hidden::hide(s.to_string())).collect());
 
         match CipherSeed::from_mnemonic(&seed_words, None) {
             Ok(seed) => break Ok(seed),
@@ -66,7 +69,7 @@ pub fn prompt_private_key_from_seed_words() -> Result<CipherSeed, ExitError> {
 }
 
 /// Return seed matching the seed words.
-pub fn get_seed_from_seed_words(seed_words: Vec<String>) -> Result<CipherSeed, ExitError> {
+pub fn get_seed_from_seed_words(seed_words: SeedWords) -> Result<CipherSeed, ExitError> {
     debug!(target: LOG_TARGET, "Return seed derived from the provided seed words");
     match CipherSeed::from_mnemonic(&seed_words, None) {
         Ok(seed) => Ok(seed),

applications/tari_merge_mining_proxy/Cargo.toml

@@ -4,7 +4,7 @@ authors = ["The Tari Development Community"]
 description = "The Tari merge mining proxy for xmrig"
 repository = "https://github.com/tari-project/tari"
 license = "BSD-3-Clause"
-version = "0.39.0"
+version = "0.40.2"
 edition = "2018"
 
 [features]
@@ -15,8 +15,8 @@ tari_common = { path = "../../common" }
 tari_comms = { path = "../../comms/core" }
 tari_core = { path = "../../base_layer/core", default-features = false, features = ["transactions"] }
 tari_app_utilities = { path = "../tari_app_utilities" }
-tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.15.7" }
-tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.7" }
+tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.16.2" }
+tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.9" }
 tari_base_node_grpc_client = {path="../../clients/rust/base_node_grpc_client" }
 tari_wallet_grpc_client = {path="../../clients/rust/wallet_grpc_client" }
 

applications/tari_miner/Cargo.toml

@@ -4,7 +4,7 @@ authors = ["The Tari Development Community"]
 description = "The tari miner implementation"
 repository = "https://github.com/tari-project/tari"
 license = "BSD-3-Clause"
-version = "0.39.0"
+version = "0.40.2"
 edition = "2018"
 
 [dependencies]
@@ -14,8 +14,8 @@ tari_common_types = { path = "../../base_layer/common_types" }
 tari_comms = { path = "../../comms/core" }
 tari_app_utilities = { path = "../tari_app_utilities" }
 tari_app_grpc = { path = "../tari_app_grpc" }
-tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.15.7" }
-tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.7" }
+tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.16.2" }
+tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.9" }
 
 crossterm = { version = "0.25.0" }
 clap = { version = "3.1.1", features = ["derive"] }

base_layer/common_types/Cargo.toml

@@ -3,14 +3,14 @@ name = "tari_common_types"
 authors = ["The Tari Development Community"]
 description = "Tari cryptocurrency common types"
 license = "BSD-3-Clause"
-version = "0.39.0"
+version = "0.40.2"
 edition = "2018"
 
 [dependencies]
-tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.15.7" }
-tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.7" }
+tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.16.2" }
+tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.9" }
 # TODO: remove this dependency and move Network into tari_common_types
-tari_common = { version = "^0.39", path = "../../common" }
+tari_common = { version = "^0.40", path = "../../common" }
 
 base64 = "0.13.0"
 digest = "0.9.0"

base_layer/core/Cargo.toml

@@ -6,7 +6,7 @@ repository = "https://github.com/tari-project/tari"
 homepage = "https://tari.com"
 readme = "README.md"
 license = "BSD-3-Clause"
-version = "0.39.0"
+version = "0.40.2"
 edition = "2018"
 
 [features]
@@ -19,21 +19,21 @@ avx2 = ["tari_crypto/simd_backend"]
 benches = ["base_node", "criterion"]
 
 [dependencies]
-tari_common = { version = "^0.39", path = "../../common" }
-tari_common_types = { version = "^0.39", path = "../../base_layer/common_types" }
-tari_comms = { version = "^0.39", path = "../../comms/core" }
-tari_comms_dht = { version = "^0.39", path = "../../comms/dht" }
-tari_comms_rpc_macros = { version = "^0.39", path = "../../comms/rpc_macros" }
-tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.15.7" }
+tari_common = { version = "^0.40", path = "../../common" }
+tari_common_types = { version = "^0.40", path = "../../base_layer/common_types" }
+tari_comms = { version = "^0.40", path = "../../comms/core" }
+tari_comms_dht = { version = "^0.40", path = "../../comms/dht" }
+tari_comms_rpc_macros = { version = "^0.40", path = "../../comms/rpc_macros" }
+tari_crypto = { git = "https://github.com/tari-project/tari-crypto.git", tag = "v0.16.2" }
 tari_metrics = { path = "../../infrastructure/metrics" }
-tari_mmr = { version = "^0.39", path = "../../base_layer/mmr", optional = true, features = ["native_bitmap"] }
-tari_p2p = { version = "^0.39", path = "../../base_layer/p2p" }
+tari_mmr = { version = "^0.40", path = "../../base_layer/mmr", optional = true, features = ["native_bitmap"] }
+tari_p2p = { version = "^0.40", path = "../../base_layer/p2p" }
 tari_script = { path = "../../infrastructure/tari_script" }
-tari_service_framework = { version = "^0.39", path = "../service_framework" }
-tari_shutdown = { version = "^0.39", path = "../../infrastructure/shutdown" }
-tari_storage = { version = "^0.39", path = "../../infrastructure/storage" }
-tari_test_utils = { version = "^0.39", path = "../../infrastructure/test_utils" }
-tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.7" }
+tari_service_framework = { version = "^0.40", path = "../service_framework" }
+tari_shutdown = { version = "^0.40", path = "../../infrastructure/shutdown" }
+tari_storage = { version = "^0.40", path = "../../infrastructure/storage" }
+tari_test_utils = { version = "^0.40", path = "../../infrastructure/test_utils" }
+tari_utilities = { git = "https://github.com/tari-project/tari_utilities.git", tag="v0.4.9" }
 
 async-trait = "0.1.50"
 bincode = "1.1.4"
@@ -76,15 +76,15 @@ tracing-attributes = "*"
 uint = { version = "0.9", default-features = false }
 
 [dev-dependencies]
-tari_p2p = { version = "^0.39", path = "../../base_layer/p2p", features = ["test-mocks"] }
-tari_test_utils = { version = "^0.39", path = "../../infrastructure/test_utils" }
+tari_p2p = { version = "^0.40", path = "../../base_layer/p2p", features = ["test-mocks"] }
+tari_test_utils = { version = "^0.40", path = "../../infrastructure/test_utils" }
 
 config = { version = "0.13.0" }
 env_logger = "0.7.0"
 tempfile = "3.1.0"
 
 [build-dependencies]
-tari_common = { version = "^0.39", path = "../../common", features = ["build"] }
+tari_common = { version = "^0.40", path = "../../common", features = ["build"] }
 
 [[bench]]
 name = "mempool"

base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs

@@ -1017,9 +1017,17 @@ impl LMDBDatabase {
         for row in inputs {
             // If input spends an output in this block, don't add it to the utxo set
             let output_hash = row.input.output_hash();
+
+            lmdb_delete(
+                txn,
+                &self.deleted_txo_mmr_position_to_height_index,
+                &row.mmr_position,
+                "deleted_txo_mmr_position_to_height_index",
+            )?;
             if output_rows.iter().any(|r| r.hash == output_hash) {
                 continue;
             }
+
             let mut input = row.input.clone();
 
             let utxo_mined_info = self.fetch_output_in_txn(txn, output_hash.as_slice())?.ok_or_else(|| {
@@ -1059,12 +1067,6 @@ impl LMDBDatabase {
                 &input.output_hash(),
                 "utxo_commitment_index",
             )?;
-            lmdb_delete(
-                txn,
-                &self.deleted_txo_mmr_position_to_height_index,
-                &row.mmr_position,
-                "deleted_txo_mmr_position_to_height_index",
-            )?;
         }
         Ok(())
     }

base_layer/core/src/transactions/coinbase_builder.rs

@@ -207,7 +207,7 @@ impl CoinbaseBuilder {
         let metadata = TransactionMetadata::new_with_features(0.into(), 0, kernel_features);
         let challenge =
             TransactionKernel::build_kernel_challenge_from_tx_meta(&public_nonce, excess.as_public_key(), &metadata);
-        let sig = Signature::sign(spending_key.clone(), nonce, &challenge)
+        let sig = Signature::sign_raw(&spending_key, nonce, &challenge)
             .map_err(|_| CoinbaseBuildError::BuildError("Challenge could not be represented as a scalar".into()))?;
 
         let hasher =
@@ -554,7 +554,7 @@ mod test {
             &KernelFeatures::empty(),
             &None,
         );
-        coinbase_kernel2.excess_sig = Signature::sign(output.spending_key, p2.nonce, &challenge).unwrap();
+        coinbase_kernel2.excess_sig = Signature::sign_raw(&output.spending_key, p2.nonce, &challenge).unwrap();
 
         tx.body.add_output(coinbase2);
         tx.body.add_kernel(coinbase_kernel2);

base_layer/core/src/transactions/test_helpers.rs

@@ -279,7 +279,7 @@ pub fn create_signature(k: PrivateKey, fee: MicroTari, lock_height: u64, feature
         &PublicKey::from_secret_key(&k),
         &tx_meta,
     );
-    Signature::sign(k, r, &e).unwrap()
+    Signature::sign_raw(&k, r, &e).unwrap()
 }
 
 /// Generate a random transaction signature given a key, returning the public key (excess) and the signature.
@@ -294,7 +294,7 @@ pub fn create_random_signature_from_s_key(
     let p = PK::from_secret_key(&s_key);
     let tx_meta = TransactionMetadata::new_with_features(fee, lock_height, features);
     let e = TransactionKernel::build_kernel_challenge_from_tx_meta(&PublicKey::from_secret_key(&r), &p, &tx_meta);
-    (p, Signature::sign(s_key, r, &e).unwrap())
+    (p, Signature::sign_raw(&s_key, r, &e).unwrap())
 }
 
 pub fn create_consensus_manager() -> ConsensusManager {

base_layer/core/src/transactions/transaction_components/transaction_kernel.rs

@@ -276,8 +276,8 @@ mod tests {
             MicroTari::from(100),
             123,
             test_params.commit_value(321.into()),
-            Signature::sign(
-                test_params.spend_key.clone(),
+            Signature::sign_raw(
+                &test_params.spend_key,
                 test_params.nonce.clone(),
                 test_params.nonce.as_bytes(),
             )

base_layer/core/src/transactions/transaction_protocol/recipient.rs

@@ -264,7 +264,7 @@ mod test {
         let r_sum = &msg.public_nonce + &p.public_nonce;
         let excess = &msg.public_excess + &PublicKey::from_secret_key(&p.spend_key);
         let e = TransactionKernel::build_kernel_challenge_from_tx_meta(&r_sum, &excess, &m);
-        let s = Signature::sign(p.spend_key.clone(), p.nonce, &e).unwrap();
+        let s = Signature::sign_raw(&p.spend_key, p.nonce, &e).unwrap();
         assert_eq!(data.partial_signature, s);
     }