fix!: prune mode sync (#5124)

Description
---
Fixes the syncing of pruned nodes

Motivation and Context
---
The utxo and kernel keys are stored as LE bytes, but when reading with the order, they need to be BE bytes to return them in the same order as they were written. When we retrieve these for normal block sync, we call `.sort()` on them with the construction of the block so the order does not matter. 
But prune mode streams the utxo and kernels via MMR positions. For this, the order is important and needs to be returned in the same order as requested. 

This broke to do no cucumber tests testing prune mode sync. This should be fixed when enabling pruned mode. 

How Has This Been Tested?
---
Manual and new unit tests. 

Fixes: #5099

---
BREAKING CHANGE: REQUIRES THAT BASE NODES RESYNC AND RECALCULATE DB KEYS

base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs

@@ -580,7 +580,7 @@ impl LMDBDatabase {
         let output_hash = output.hash();
         let witness_hash = output.witness_hash();
 
-        let output_key = OutputKey::try_from_parts(&[header_hash.as_slice(), mmr_position.to_le_bytes().as_slice()])?;
+        let output_key = OutputKey::try_from_parts(&[header_hash.as_slice(), mmr_position.to_be_bytes().as_slice()])?;
 
         lmdb_insert(
             txn,
@@ -632,7 +632,7 @@ impl LMDBDatabase {
                 header_hash.to_hex(),
             )));
         }
-        let key = OutputKey::try_from_parts(&[header_hash.as_slice(), mmr_position.to_le_bytes().as_slice()])?;
+        let key = OutputKey::try_from_parts(&[header_hash.as_slice(), mmr_position.to_be_bytes().as_slice()])?;
         lmdb_insert(
             txn,
             &*self.txos_hash_to_index_db,
@@ -667,7 +667,7 @@ impl LMDBDatabase {
         let hash = kernel.hash();
         let key = KernelKey::try_from_parts(&[
             header_hash.as_slice(),
-            mmr_position.to_le_bytes().as_slice(),
+            mmr_position.to_be_bytes().as_slice(),
             hash.as_slice(),
         ])?;
 
@@ -734,7 +734,7 @@ impl LMDBDatabase {
         let hash = input.canonical_hash();
         let key = InputKey::try_from_parts(&[
             header_hash.as_slice(),
-            mmr_position.to_le_bytes().as_slice(),
+            mmr_position.to_be_bytes().as_slice(),
             hash.as_slice(),
         ])?;
         lmdb_insert(
@@ -1537,7 +1537,7 @@ impl LMDBDatabase {
                 &u64::from(pos + 1).to_be_bytes(),
             )
             .or_not_found("BlockHeader", "mmr_position", pos.to_string())?;
-            let key = OutputKey::try_from_parts(&[hash.as_slice(), pos.to_le_bytes().as_slice()])?;
+            let key = OutputKey::try_from_parts(&[hash.as_slice(), pos.to_be_bytes().as_slice()])?;
             debug!(target: LOG_TARGET, "Pruning output: {}", key);
             self.prune_output(write_txn, &key)?;
         }
@@ -2031,7 +2031,7 @@ impl BlockchainBackend for LMDBDatabase {
         {
             let key = KernelKey::try_from_parts(&[
                 header_hash.as_slice(),
-                mmr_position.to_le_bytes().as_slice(),
+                mmr_position.to_be_bytes().as_slice(),
                 hash.as_slice(),
             ])?;
             Ok(lmdb_get(&txn, &self.kernels_db, &key)?

base_layer/core/tests/chain_storage_tests/chain_backend.rs

@@ -20,13 +20,36 @@
 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+use rand::rngs::OsRng;
 use tari_common::configuration::Network;
+use tari_common_types::types::{
+    ComAndPubSignature,
+    Commitment,
+    FixedHash,
+    PrivateKey,
+    PublicKey,
+    RangeProof,
+    Signature,
+};
 use tari_core::{
+    blocks::{BlockAccumulatedData, BlockHeader, BlockHeaderAccumulatedData, ChainHeader, UpdateBlockAccumulatedData},
     chain_storage::{create_lmdb_database, BlockchainBackend, ChainStorageError, DbKey, DbTransaction, DbValue},
     consensus::{ConsensusManager, ConsensusManagerBuilder},
+    covenants::Covenant,
     test_helpers::blockchain::create_test_db,
+    transactions::transaction_components::{
+        EncryptedValue,
+        KernelFeatures,
+        OutputFeatures,
+        TransactionKernel,
+        TransactionKernelVersion,
+        TransactionOutput,
+        TransactionOutputVersion,
+    },
     tx,
 };
+use tari_crypto::keys::{PublicKey as PKtrait, SecretKey as SKtrait};
+use tari_script::TariScript;
 use tari_storage::lmdb_store::LMDBConfig;
 use tari_test_utils::paths::create_temporary_data_path;
 
@@ -62,6 +85,105 @@ fn test_lmdb_insert_contains_delete_and_fetch_orphan() {
     assert!(!db.contains(&DbKey::OrphanBlock(hash)).unwrap());
 }
 
+#[test]
+fn test_kernel_order() {
+    let mut db = create_test_db();
+
+    let block_hash = FixedHash::zero();
+    let mut kernels = Vec::with_capacity(2000);
+    let version = TransactionKernelVersion::V0;
+    let features = KernelFeatures::default();
+    for _i in 0..2000 {
+        let pvt_key = PrivateKey::random(&mut OsRng);
+        let pub_key = PublicKey::from_secret_key(&pvt_key);
+        let commitment = Commitment::from_public_key(&pub_key);
+        let sig = Signature::new(pub_key, pvt_key);
+        let kernel = TransactionKernel::new(version, features, 0.into(), 0, commitment, sig, None);
+        kernels.push(kernel);
+    }
+    kernels.sort();
+
+    for (i, kernel) in kernels.iter().enumerate().take(2000) {
+        let mut tx = DbTransaction::new();
+        tx.insert_kernel(kernel.clone(), block_hash, i as u32);
+        db.write(tx).unwrap();
+    }
+
+    let read_kernels = db.fetch_kernels_in_block(&block_hash).unwrap();
+    assert_eq!(kernels.len(), read_kernels.len());
+    for i in 0..2000 {
+        assert_eq!(kernels[i], read_kernels[i]);
+    }
+}
+
+#[test]
+fn test_utxo_order() {
+    let mut db = create_test_db();
+
+    let block_data = BlockAccumulatedData::default();
+    let header = BlockHeader::new(0);
+    let block_hash = header.hash();
+    let mut utxos = Vec::with_capacity(2000);
+    let version = TransactionOutputVersion::V0;
+    let features = OutputFeatures::default();
+    let script = TariScript::default();
+    let proof = RangeProof::default();
+    let sig = ComAndPubSignature::default();
+    let covenant = Covenant::default();
+    let encrypt = EncryptedValue::default();
+    for _i in 0..2000 {
+        let pvt_key = PrivateKey::random(&mut OsRng);
+        let pub_key = PublicKey::from_secret_key(&pvt_key);
+        let commitment = Commitment::from_public_key(&pub_key);
+        let utxo = TransactionOutput::new(
+            version,
+            features.clone(),
+            commitment,
+            proof.clone(),
+            script.clone(),
+            pub_key,
+            sig.clone(),
+            covenant.clone(),
+            encrypt.clone(),
+            0.into(),
+        );
+        utxos.push(utxo);
+    }
+    utxos.sort();
+
+    for (i, utxo) in utxos.iter().enumerate().take(2000) {
+        let mut tx = DbTransaction::new();
+        tx.insert_utxo(utxo.clone(), block_hash, 0, i as u32, 0);
+        db.write(tx).unwrap();
+    }
+
+    let mut tx = DbTransaction::new();
+    let data = BlockHeaderAccumulatedData {
+        hash: header.hash(),
+        ..Default::default()
+    };
+    let chainheader = ChainHeader::try_construct(header, data).unwrap();
+    let sum = block_data.kernel_sum().clone();
+    let (kernels, utxo_set, witness, deleted) = block_data.dissolve();
+    let update_data = UpdateBlockAccumulatedData {
+        kernel_hash_set: Some(kernels),
+        utxo_hash_set: Some(utxo_set),
+        witness_hash_set: Some(witness),
+        deleted_diff: Some(deleted.into()),
+        kernel_sum: Some(sum),
+    };
+    tx.insert_chain_header(chainheader);
+    tx.update_block_accumulated_data(block_hash, update_data);
+
+    db.write(tx).unwrap();
+
+    let read_utxos = db.fetch_utxos_in_block(&block_hash, None).unwrap().0;
+    assert_eq!(utxos.len(), read_utxos.len());
+    for i in 0..2000 {
+        assert_eq!(&utxos[i], read_utxos[i].as_transaction_output().unwrap());
+    }
+}
+
 #[test]
 fn test_lmdb_file_lock() {
     // Create temporary test folder