fix: fix flakey rust tests (#3435)

Description
---
This PR aims to fix the following flakey tests:
1. `test_txo_validation`
2. `test_transaction_resending`

The fix for 1 was to put in an explicit wait for the validation protocol to complete, on the slower CI system is seems that the validation protocol was slow to update the database state before the main test queried the balance.

The fix for 2 is to increase the cool down and resend times to make them more resilient to the slower CI environment. The current times were quite tight which was fine on a local dev machine. This might still prove to be flakey in the future.

How Has This Been Tested?
---
These are fixes to tests

base_layer/wallet/tests/output_manager_service/service.rs

@@ -69,7 +69,7 @@ use tari_wallet::{
     output_manager_service::{
         config::OutputManagerServiceConfig,
         error::{OutputManagerError, OutputManagerStorageError},
-        handle::OutputManagerHandle,
+        handle::{OutputManagerEvent, OutputManagerHandle},
         service::OutputManagerService,
         storage::{
             database::{OutputManagerBackend, OutputManagerDatabase},
@@ -82,6 +82,7 @@ use tari_wallet::{
 use tokio::{
     sync::{broadcast, broadcast::channel},
     task,
+    time::sleep,
 };
 
 #[allow(clippy::type_complexity)]
@@ -1347,7 +1348,9 @@ async fn test_txo_validation() {
     query_deleted_response.best_block = [8u8; 16].to_vec();
     rpc_service_state.set_query_deleted_response(query_deleted_response);
 
-    oms.validate_txos().await.unwrap();
+    let mut event_stream = oms.get_event_stream();
+
+    let validation_id = oms.validate_txos().await.unwrap();
 
     let _utxo_query_calls = rpc_service_state
         .wait_pop_utxo_query_calls(1, Duration::from_secs(60))
@@ -1359,6 +1362,26 @@ async fn test_txo_validation() {
         .await
         .unwrap();
 
+    let delay = sleep(Duration::from_secs(30));
+    tokio::pin!(delay);
+    let mut validation_completed = false;
+    loop {
+        tokio::select! {
+            event = event_stream.recv() => {
+                 if let OutputManagerEvent::TxoValidationSuccess(id) = &*event.unwrap(){
+                    if id == &validation_id {
+                        validation_completed = true;
+                        break;
+                    }
+                }
+            },
+            () = &mut delay => {
+                break;
+            },
+        }
+    }
+    assert!(validation_completed, "Validation protocol should complete");
+
     let balance = oms.get_balance().await.unwrap();
     assert_eq!(
         balance.available_balance,

base_layer/wallet/tests/transaction_service/service.rs

@@ -3762,8 +3762,8 @@ fn test_transaction_resending() {
         factories.clone(),
         connection,
         Some(TransactionServiceConfig {
-            transaction_resend_period: Duration::from_secs(10),
-            resend_response_cooldown: Duration::from_secs(5),
+            transaction_resend_period: Duration::from_secs(20),
+            resend_response_cooldown: Duration::from_secs(10),
             ..Default::default()
         }),
     );
@@ -3786,7 +3786,7 @@ fn test_transaction_resending() {
 
     // Check that there were repeats
     alice_outbound_service
-        .wait_call_count(2, Duration::from_secs(30))
+        .wait_call_count(2, Duration::from_secs(60))
         .expect("Alice call wait 1");
 
     let mut alice_sender_message = TransactionSenderMessage::None;
@@ -3824,8 +3824,8 @@ fn test_transaction_resending() {
         factories,
         connection,
         Some(TransactionServiceConfig {
-            transaction_resend_period: Duration::from_secs(10),
-            resend_response_cooldown: Duration::from_secs(5),
+            transaction_resend_period: Duration::from_secs(20),
+            resend_response_cooldown: Duration::from_secs(10),
             ..Default::default()
         }),
     );
@@ -3840,7 +3840,7 @@ fn test_transaction_resending() {
 
     // Check that the reply was repeated
     bob_outbound_service
-        .wait_call_count(2, Duration::from_secs(30))
+        .wait_call_count(2, Duration::from_secs(60))
         .expect("Bob call wait 1");
 
     let mut bob_reply_message;
@@ -3861,16 +3861,16 @@ fn test_transaction_resending() {
 
     assert!(bob_outbound_service.wait_call_count(1, Duration::from_secs(2)).is_err());
 
-    // Wait for the cooldown to expire but before the resend period has elapsed see if a repeat illicts a reponse.
-    runtime.block_on(async { sleep(Duration::from_secs(2)).await });
+    // Wait for the cooldown to expire but before the resend period has elapsed see if a repeat illicits a response.
+    runtime.block_on(async { sleep(Duration::from_secs(8)).await });
     runtime
         .block_on(bob_tx_sender.send(create_dummy_message(
             alice_sender_message.into(),
             alice_node_identity.public_key(),
         )))
         .unwrap();
     bob_outbound_service
-        .wait_call_count(2, Duration::from_secs(30))
+        .wait_call_count(2, Duration::from_secs(60))
         .expect("Bob call wait 2");
     let _ = bob_outbound_service.pop_call().unwrap();
     let call = bob_outbound_service.pop_call().unwrap();
@@ -3888,7 +3888,7 @@ fn test_transaction_resending() {
         .unwrap();
 
     alice_outbound_service
-        .wait_call_count(2, Duration::from_secs(30))
+        .wait_call_count(2, Duration::from_secs(60))
         .expect("Alice call wait 2");
 
     let _ = alice_outbound_service.pop_call().unwrap();
@@ -3905,11 +3905,11 @@ fn test_transaction_resending() {
         .unwrap();
 
     assert!(alice_outbound_service
-        .wait_call_count(1, Duration::from_secs(4))
+        .wait_call_count(1, Duration::from_secs(5))
         .is_err());
 
-    // Wait for the cooldown to expire but before the resend period has elapsed see if a repeat illicts a reponse.
-    runtime.block_on(async { sleep(Duration::from_secs(2)).await });
+    // Wait for the cooldown to expire but before the resend period has elapsed see if a repeat illicts a response.
+    runtime.block_on(async { sleep(Duration::from_secs(6)).await });
 
     runtime
         .block_on(alice_tx_reply_sender.send(create_dummy_message(