fix!: don't include duplicate unique ID in same block template (#3713)

Description
---

- Keeps track of unique IDs in the current unconfirmed transaction set
- Mempool will not select unique IDs for block templates
- Add basic mempool unit test
- Implement consensus encoding for remaining output features fields/structs (consensus breaking)
- Update cucumber JS encoding and allow unique id to be set for transactions
- Add cucumber test 
- Adds new unit tests for output features consensus encoding

Motivation and Context
---
A duplicate unique ID in the same block is invalid and will be rejected by the network, since nothing has changed in the mempool after the rejected block the same transactions could be selected for the next block and so on. For this reason, the base node should not place transactions with duplicate unique IDs in the block template.

How Has This Been Tested?
---
New unit and cucumber test

base_layer/core/src/base_node/service/service.rs

@@ -645,7 +645,5 @@ async fn handle_incoming_block<B: BlockchainBackend + 'static>(
         .handle_new_block_message(new_block, source_peer.node_id)
         .await?;
 
-    // TODO - retain peer info for stats and potential banning for sending invalid blocks
-
     Ok(())
 }

base_layer/core/src/consensus/consensus_encoding.rs

@@ -20,8 +20,20 @@
 //  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 //  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+mod bytes;
+mod crypto;
+mod generic;
+mod integers;
+mod script;
+mod vec;
+
 use std::io;
 
+pub use vec::MaxSizeVec;
+
+pub use self::bytes::MaxSizeBytes;
+use crate::common::byte_counter::ByteCounter;
+
 /// Abstracts the ability of a type to canonically encode itself for the purposes of consensus
 pub trait ConsensusEncoding {
     /// Encode to the given writer returning the number of bytes written.
@@ -32,7 +44,12 @@ pub trait ConsensusEncoding {
 pub trait ConsensusEncodingSized: ConsensusEncoding {
     /// The return value MUST be the exact byte size of the implementing type
     /// and SHOULD be implemented without allocations.
-    fn consensus_encode_exact_size(&self) -> usize;
+    fn consensus_encode_exact_size(&self) -> usize {
+        let mut byte_counter = ByteCounter::new();
+        self.consensus_encode(&mut byte_counter)
+            .expect("ByteCounter is infallible");
+        byte_counter.get()
+    }
 }
 
 /// Abstracts the ability of a type to be decoded from canonical consensus bytes
@@ -54,201 +71,19 @@ impl<T: ConsensusEncoding + ConsensusEncodingSized + ?Sized> ToConsensusBytes fo
     }
 }
 
-impl ConsensusEncoding for Vec<u8> {
-    fn consensus_encode<W: io::Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
-        writer.write(self)
-    }
-}
-
-impl ConsensusEncodingSized for Vec<u8> {
-    fn consensus_encode_exact_size(&self) -> usize {
-        self.len()
-    }
-}
-
-macro_rules! consensus_encoding_varint_impl {
-    ($ty:ty) => {
-        impl ConsensusEncoding for $ty {
-            fn consensus_encode<W: io::Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
-                use integer_encoding::VarIntWriter;
-                let bytes_written = writer.write_varint(*self)?;
-                Ok(bytes_written)
-            }
-        }
-
-        impl ConsensusDecoding for $ty {
-            fn consensus_decode<R: io::Read>(reader: &mut R) -> Result<Self, io::Error> {
-                use integer_encoding::VarIntReader;
-                let value = reader.read_varint()?;
-                Ok(value)
-            }
-        }
-
-        impl ConsensusEncodingSized for $ty {
-            fn consensus_encode_exact_size(&self) -> usize {
-                use integer_encoding::VarInt;
-                self.required_space()
-            }
-        }
-    };
-}
-
-consensus_encoding_varint_impl!(u8);
-consensus_encoding_varint_impl!(u64);
-
-// Keep separate the dependencies of the impls that may in future be implemented in tari crypto
-mod impls {
-    use std::io::Read;
-
-    use tari_common_types::types::{Commitment, PrivateKey, PublicKey, Signature};
-    use tari_crypto::{
-        keys::{PublicKey as PublicKeyTrait, SecretKey as SecretKeyTrait},
-        script::{ExecutionStack, TariScript},
-    };
-    use tari_utilities::ByteArray;
-
+#[cfg(test)]
+pub mod test {
     use super::*;
-    use crate::common::byte_counter::ByteCounter;
-
-    //---------------------------------- TariScript --------------------------------------------//
-
-    impl ConsensusEncoding for TariScript {
-        fn consensus_encode<W: io::Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
-            self.as_bytes().consensus_encode(writer)
-        }
-    }
-
-    impl ConsensusEncodingSized for TariScript {
-        fn consensus_encode_exact_size(&self) -> usize {
-            let mut counter = ByteCounter::new();
-            // TODO: consensus_encode_exact_size must be cheap to run
-            // unreachable panic: ByteCounter is infallible
-            self.consensus_encode(&mut counter).expect("unreachable");
-            counter.get()
-        }
-    }
-
-    impl ConsensusEncoding for ExecutionStack {
-        fn consensus_encode<W: io::Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
-            self.as_bytes().consensus_encode(writer)
-        }
-    }
-
-    impl ConsensusEncodingSized for ExecutionStack {
-        fn consensus_encode_exact_size(&self) -> usize {
-            let mut counter = ByteCounter::new();
-            // TODO: consensus_encode_exact_size must be cheap to run
-            // unreachable panic: ByteCounter is infallible
-            self.consensus_encode(&mut counter).expect("unreachable");
-            counter.get()
-        }
-    }
-
-    //---------------------------------- PublicKey --------------------------------------------//
-
-    impl ConsensusEncoding for PublicKey {
-        fn consensus_encode<W: io::Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
-            writer.write(self.as_bytes())
-        }
-    }
-
-    impl ConsensusEncodingSized for PublicKey {
-        fn consensus_encode_exact_size(&self) -> usize {
-            PublicKey::key_length()
-        }
-    }
-
-    impl ConsensusDecoding for PublicKey {
-        fn consensus_decode<R: Read>(reader: &mut R) -> Result<Self, io::Error> {
-            let mut buf = [0u8; 32];
-            reader.read_exact(&mut buf)?;
-            let pk = PublicKey::from_bytes(&buf[..]).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?;
-            Ok(pk)
-        }
-    }
-
-    //---------------------------------- PrivateKey --------------------------------------------//
-
-    impl ConsensusEncoding for PrivateKey {
-        fn consensus_encode<W: io::Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
-            writer.write(self.as_bytes())
-        }
-    }
-
-    impl ConsensusEncodingSized for PrivateKey {
-        fn consensus_encode_exact_size(&self) -> usize {
-            PrivateKey::key_length()
-        }
-    }
-
-    impl ConsensusDecoding for PrivateKey {
-        fn consensus_decode<R: Read>(reader: &mut R) -> Result<Self, io::Error> {
-            let mut buf = [0u8; 32];
-            reader.read_exact(&mut buf)?;
-            let sk =
-                PrivateKey::from_bytes(&buf[..]).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?;
-            Ok(sk)
-        }
-    }
-
-    //---------------------------------- Commitment --------------------------------------------//
-
-    impl ConsensusEncoding for Commitment {
-        fn consensus_encode<W: io::Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
-            let buf = self.as_bytes();
-            let len = buf.len();
-            writer.write_all(buf)?;
-            Ok(len)
-        }
-    }
-
-    impl ConsensusEncodingSized for Commitment {
-        fn consensus_encode_exact_size(&self) -> usize {
-            32
-        }
-    }
-
-    impl ConsensusDecoding for Commitment {
-        fn consensus_decode<R: Read>(reader: &mut R) -> Result<Self, io::Error> {
-            let mut buf = [0u8; 32];
-            reader.read_exact(&mut buf)?;
-            let commitment =
-                Commitment::from_bytes(&buf[..]).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?;
-            Ok(commitment)
-        }
-    }
-
-    //---------------------------------- Signature --------------------------------------------//
-
-    impl ConsensusEncoding for Signature {
-        fn consensus_encode<W: io::Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
-            let pub_nonce = self.get_public_nonce().as_bytes();
-            let mut written = pub_nonce.len();
-            writer.write_all(pub_nonce)?;
-            let sig = self.get_signature().as_bytes();
-            written += sig.len();
-            writer.write_all(sig)?;
-            Ok(written)
-        }
-    }
-
-    impl ConsensusEncodingSized for Signature {
-        fn consensus_encode_exact_size(&self) -> usize {
-            self.get_signature().consensus_encode_exact_size() + self.get_public_nonce().consensus_encode_exact_size()
-        }
-    }
 
-    impl ConsensusDecoding for Signature {
-        fn consensus_decode<R: Read>(reader: &mut R) -> Result<Self, io::Error> {
-            let mut buf = [0u8; 32];
-            reader.read_exact(&mut buf)?;
-            let pub_nonce =
-                PublicKey::from_bytes(&buf[..]).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?;
-            let mut buf = [0u8; 32];
-            reader.read_exact(&mut buf)?;
-            let sig =
-                PrivateKey::from_bytes(&buf[..]).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?;
-            Ok(Signature::new(pub_nonce, sig))
-        }
+    /// Test utility function that checks the correctness of the ConsensusEncoding, ConsensusEncodingSized,
+    /// ConsensusDecoding implementations
+    pub fn check_consensus_encoding_correctness<T>(subject: T) -> Result<(), io::Error>
+    where T: ConsensusEncoding + ConsensusEncodingSized + ConsensusDecoding + Eq + std::fmt::Debug {
+        let mut buf = Vec::new();
+        subject.consensus_encode(&mut buf)?;
+        assert_eq!(buf.len(), subject.consensus_encode_exact_size());
+        let decoded = T::consensus_decode(&mut buf.as_slice())?;
+        assert_eq!(decoded, subject);
+        Ok(())
     }
 }

base_layer/core/src/consensus/consensus_encoding/bytes.rs

@@ -0,0 +1,108 @@
+//  Copyright 2022, The Tari Project
+//
+//  Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+//  following conditions are met:
+//
+//  1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
+//  disclaimer.
+//
+//  2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
+//  following disclaimer in the documentation and/or other materials provided with the distribution.
+//
+//  3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
+//  products derived from this software without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+//  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+//  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+//  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+//  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+//  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+use std::{
+    io,
+    io::{Error, Read, Write},
+};
+
+use integer_encoding::{VarInt, VarIntReader, VarIntWriter};
+
+use crate::consensus::{ConsensusDecoding, ConsensusEncoding, ConsensusEncodingSized};
+
+impl ConsensusEncoding for Vec<u8> {
+    fn consensus_encode<W: Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
+        let len = self.len();
+        let mut written = writer.write_varint(len)?;
+        written += writer.write(self)?;
+        Ok(written)
+    }
+}
+
+impl ConsensusEncodingSized for Vec<u8> {
+    fn consensus_encode_exact_size(&self) -> usize {
+        let len = self.len();
+        len.required_space() + len
+    }
+}
+
+pub struct MaxSizeBytes<const MAX: usize> {
+    inner: Vec<u8>,
+}
+
+impl<const MAX: usize> From<MaxSizeBytes<MAX>> for Vec<u8> {
+    fn from(value: MaxSizeBytes<MAX>) -> Self {
+        value.inner
+    }
+}
+
+impl<const MAX: usize> ConsensusDecoding for MaxSizeBytes<MAX> {
+    fn consensus_decode<R: Read>(reader: &mut R) -> Result<Self, Error> {
+        let len = reader.read_varint()?;
+        if len > MAX {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                format!("Vec size ({}) exceeded maximum ({})", len, MAX),
+            ));
+        }
+        let mut bytes = vec![0u8; len];
+        reader.read_exact(&mut bytes)?;
+        Ok(Self { inner: bytes })
+    }
+}
+
+impl ConsensusEncoding for &[u8] {
+    fn consensus_encode<W: Write>(&self, writer: &mut W) -> Result<usize, io::Error> {
+        let len = self.len();
+        let mut written = writer.write_varint(len)?;
+        written += writer.write(self)?;
+        Ok(written)
+    }
+}
+
+impl ConsensusEncodingSized for &[u8] {
+    fn consensus_encode_exact_size(&self) -> usize {
+        self.len()
+    }
+}
+
+impl<const N: usize> ConsensusEncoding for [u8; N] {
+    fn consensus_encode<W: Write>(&self, writer: &mut W) -> Result<usize, Error> {
+        // For fixed length types we dont need a length byte
+        writer.write_all(&self[..])?;
+        Ok(N)
+    }
+}
+
+impl<const N: usize> ConsensusEncodingSized for [u8; N] {
+    fn consensus_encode_exact_size(&self) -> usize {
+        N
+    }
+}
+
+impl<const N: usize> ConsensusDecoding for [u8; N] {
+    fn consensus_decode<R: Read>(reader: &mut R) -> Result<Self, Error> {
+        let mut buf = [0u8; N];
+        reader.read_exact(&mut buf)?;
+        Ok(buf)
+    }
+}