adds shred::layout::get_signed_data (backport #29438) (#29854)

* adds shred::layout::get_signed_data (#29438)

Working towards removing merkle root from shreds payload, the commit
implements api to obtain signed data from shreds binary.

(cherry picked from commit 50afb80)

# Conflicts:
#	ledger/src/shred.rs

* resolves mergify merge conflicts

Co-authored-by: behzad nouri <[email protected]>

ledger/src/shred.rs

@@ -309,7 +309,7 @@ use dispatch;
 impl Shred {
     dispatch!(fn common_header(&self) -> &ShredCommonHeader);
     dispatch!(fn set_signature(&mut self, signature: Signature));
-    dispatch!(fn signed_message(&self) -> &[u8]);
+    dispatch!(fn signed_data(&self) -> &[u8]);
 
     // Returns the portion of the shred's payload which is erasure coded.
     dispatch!(pub(crate) fn erasure_shard(self) -> Result<Vec<u8>, Error>);
@@ -460,7 +460,7 @@ impl Shred {
     }
 
     pub fn sign(&mut self, keypair: &Keypair) {
-        let signature = keypair.sign_message(self.signed_message());
+        let signature = keypair.sign_message(self.signed_data());
         self.set_signature(signature);
     }
 
@@ -508,8 +508,8 @@ impl Shred {
 
     #[must_use]
     pub fn verify(&self, pubkey: &Pubkey) -> bool {
-        let message = self.signed_message();
-        self.signature().verify(pubkey.as_ref(), message)
+        let data = self.signed_data();
+        self.signature().verify(pubkey.as_ref(), data)
     }
 
     // Returns true if the erasure coding of the two shreds mismatch.
@@ -538,9 +538,26 @@ impl Shred {
 // Helper methods to extract pieces of the shred from the payload
 // without deserializing the entire payload.
 pub mod layout {
+    use {super::*, crate::shred::merkle::MerkleRoot, std::ops::Range};
     #[cfg(test)]
-    use crate::shred::merkle::MerkleRoot;
-    use {super::*, std::ops::Range};
+    use {
+        rand::{seq::SliceRandom, Rng},
+        std::collections::HashMap,
+    };
+
+    pub(crate) enum SignedData<'a> {
+        Chunk(&'a [u8]),
+        MerkleRoot(MerkleRoot),
+    }
+
+    impl<'a> AsRef<[u8]> for SignedData<'a> {
+        fn as_ref(&self) -> &[u8] {
+            match self {
+                Self::Chunk(chunk) => chunk,
+                Self::MerkleRoot(root) => root,
+            }
+        }
+    }
 
     fn get_shred_size(packet: &Packet) -> Option<usize> {
         let size = packet.data(..)?.len();
@@ -615,18 +632,36 @@ pub mod layout {
         ))
     }
 
-    // Returns slice range of the shred payload which is signed.
-    pub(crate) fn get_signed_message_range(shred: &[u8]) -> Option<Range<usize>> {
-        let range = match get_shred_variant(shred).ok()? {
-            ShredVariant::LegacyCode | ShredVariant::LegacyData => legacy::SIGNED_MESSAGE_RANGE,
+    pub(crate) fn get_signed_data(shred: &[u8]) -> Option<SignedData> {
+        let data = match get_shred_variant(shred).ok()? {
+            ShredVariant::LegacyCode | ShredVariant::LegacyData => {
+                let chunk = shred.get(self::legacy::SIGNED_MESSAGE_OFFSETS)?;
+                SignedData::Chunk(chunk)
+            }
+            ShredVariant::MerkleCode(proof_size) => {
+                let merkle_root = self::merkle::ShredCode::get_merkle_root(shred, proof_size)?;
+                SignedData::MerkleRoot(merkle_root)
+            }
+            ShredVariant::MerkleData(proof_size) => {
+                let merkle_root = self::merkle::ShredData::get_merkle_root(shred, proof_size)?;
+                SignedData::MerkleRoot(merkle_root)
+            }
+        };
+        Some(data)
+    }
+
+    // Returns offsets within the shred payload which is signed.
+    pub(crate) fn get_signed_data_offsets(shred: &[u8]) -> Option<Range<usize>> {
+        let offsets = match get_shred_variant(shred).ok()? {
+            ShredVariant::LegacyCode | ShredVariant::LegacyData => legacy::SIGNED_MESSAGE_OFFSETS,
             ShredVariant::MerkleCode(proof_size) => {
-                merkle::ShredCode::get_signed_message_range(proof_size)?
+                merkle::ShredCode::get_signed_data_offsets(proof_size)?
             }
             ShredVariant::MerkleData(proof_size) => {
-                merkle::ShredData::get_signed_message_range(proof_size)?
+                merkle::ShredData::get_signed_data_offsets(proof_size)?
             }
         };
-        (range.end <= shred.len()).then(|| range)
+        (offsets.end <= shred.len()).then(|| offsets)
     }
 
     pub(crate) fn get_reference_tick(shred: &[u8]) -> Result<u8, Error> {
@@ -652,6 +687,62 @@ pub mod layout {
             }
         }
     }
+
+    // Minimally corrupts the packet so that the signature no longer verifies.
+    #[cfg(test)]
+    pub(crate) fn corrupt_packet<R: Rng>(
+        rng: &mut R,
+        packet: &mut Packet,
+        keypairs: &HashMap<Slot, Keypair>,
+    ) {
+        fn modify_packet<R: Rng>(rng: &mut R, packet: &mut Packet, offsets: Range<usize>) {
+            let buffer = packet.buffer_mut();
+            let byte = buffer[offsets].choose_mut(rng).unwrap();
+            *byte = rng.gen::<u8>().max(1u8).wrapping_add(*byte);
+        }
+        let shred = get_shred(packet).unwrap();
+        let merkle_proof_size = match get_shred_variant(shred).unwrap() {
+            ShredVariant::LegacyCode | ShredVariant::LegacyData => None,
+            ShredVariant::MerkleCode(proof_size) | ShredVariant::MerkleData(proof_size) => {
+                Some(proof_size)
+            }
+        };
+        let coin_flip: bool = rng.gen();
+        if coin_flip {
+            // Corrupt one byte within the signature offsets.
+            modify_packet(rng, packet, 0..SIGNATURE_BYTES);
+        } else {
+            // Corrupt one byte within the signed data offsets.
+            let size = shred.len();
+            let offsets = get_signed_data_offsets(shred).unwrap();
+            modify_packet(rng, packet, offsets);
+            if let Some(proof_size) = merkle_proof_size {
+                // Also need to corrupt the merkle proof.
+                // Proof entries are each 20 bytes at the end of shreds.
+                let offset = usize::from(proof_size) * 20;
+                modify_packet(rng, packet, size - offset..size);
+            }
+        }
+        // Assert that the signature no longer verifies.
+        let shred = get_shred(packet).unwrap();
+        let slot = get_slot(shred).unwrap();
+        let signature = get_signature(shred).unwrap();
+        if coin_flip {
+            let pubkey = keypairs[&slot].pubkey();
+            let data = get_signed_data(shred).unwrap();
+            assert!(!signature.verify(pubkey.as_ref(), data.as_ref()));
+            let offsets = get_signed_data_offsets(shred).unwrap();
+            assert!(!signature.verify(pubkey.as_ref(), &shred[offsets]));
+        } else {
+            // Slot may have been corrupted and no longer mapping to a keypair.
+            let pubkey = keypairs.get(&slot).map(Keypair::pubkey).unwrap_or_default();
+            if let Some(data) = get_signed_data(shred) {
+                assert!(!signature.verify(pubkey.as_ref(), data.as_ref()));
+            }
+            let offsets = get_signed_data_offsets(shred).unwrap_or_default();
+            assert!(!signature.verify(pubkey.as_ref(), &shred[offsets]));
+        }
+    }
 }
 
 impl From<ShredCode> for Shred {

ledger/src/shred/legacy.rs

@@ -15,7 +15,8 @@ use {
 
 // All payload including any zero paddings are signed.
 // Code and data shreds have the same payload size.
-pub(super) const SIGNED_MESSAGE_RANGE: Range<usize> = SIZE_OF_SIGNATURE..ShredData::SIZE_OF_PAYLOAD;
+pub(super) const SIGNED_MESSAGE_OFFSETS: Range<usize> =
+    SIZE_OF_SIGNATURE..ShredData::SIZE_OF_PAYLOAD;
 const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, ShredCode::SIZE_OF_PAYLOAD);
 const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, 1228);
 const_assert_eq!(ShredData::CAPACITY, 1051);
@@ -108,7 +109,7 @@ impl Shred for ShredData {
         shred_data::sanitize(self)
     }
 
-    fn signed_message(&self) -> &[u8] {
+    fn signed_data(&self) -> &[u8] {
         debug_assert_eq!(self.payload.len(), Self::SIZE_OF_PAYLOAD);
         &self.payload[SIZE_OF_SIGNATURE..]
     }
@@ -170,7 +171,7 @@ impl Shred for ShredCode {
         shred_code::sanitize(self)
     }
 
-    fn signed_message(&self) -> &[u8] {
+    fn signed_data(&self) -> &[u8] {
         debug_assert_eq!(self.payload.len(), Self::SIZE_OF_PAYLOAD);
         &self.payload[SIZE_OF_SIGNATURE..]
     }

ledger/src/shred/merkle.rs

@@ -91,12 +91,12 @@ impl Shred {
     dispatch!(fn sanitize(&self, verify_merkle_proof: bool) -> Result<(), Error>);
     dispatch!(fn set_merkle_branch(&mut self, merkle_branch: &MerkleBranch) -> Result<(), Error>);
     dispatch!(fn set_signature(&mut self, signature: Signature));
-    dispatch!(fn signed_message(&self) -> &[u8]);
+    dispatch!(fn signed_data(&self) -> &[u8]);
 
     #[must_use]
     fn verify(&self, pubkey: &Pubkey) -> bool {
-        let message = self.signed_message();
-        self.signature().verify(pubkey.as_ref(), message)
+        let data = self.signed_data();
+        self.signature().verify(pubkey.as_ref(), data)
     }
 
     fn signature(&self) -> Signature {
@@ -147,7 +147,7 @@ impl ShredData {
             .ok_or(Error::InvalidProofSize(proof_size))
     }
 
-    pub(super) fn get_signed_message_range(proof_size: u8) -> Option<Range<usize>> {
+    pub(super) fn get_signed_data_offsets(proof_size: u8) -> Option<Range<usize>> {
         let data_buffer_size = Self::capacity(proof_size).ok()?;
         let offset = Self::SIZE_OF_HEADERS + data_buffer_size;
         Some(offset..offset + SIZE_OF_MERKLE_ROOT)
@@ -247,7 +247,6 @@ impl ShredData {
         shred_data::sanitize(self)
     }
 
-    #[cfg(test)]
     pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option<MerkleRoot> {
         debug_assert_eq!(
             shred::layout::get_shred_variant(shred).unwrap(),
@@ -332,7 +331,7 @@ impl ShredCode {
         Ok(verify_merkle_proof(index, node, &self.merkle_branch()?))
     }
 
-    pub(super) fn get_signed_message_range(proof_size: u8) -> Option<Range<usize>> {
+    pub(super) fn get_signed_data_offsets(proof_size: u8) -> Option<Range<usize>> {
         let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size).ok()?;
         Some(offset..offset + SIZE_OF_MERKLE_ROOT)
     }
@@ -397,7 +396,6 @@ impl ShredCode {
         shred_code::sanitize(self)
     }
 
-    #[cfg(test)]
     pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option<MerkleRoot> {
         debug_assert_eq!(
             shred::layout::get_shred_variant(shred).unwrap(),
@@ -488,7 +486,7 @@ impl ShredTrait for ShredData {
         self.sanitize(/*verify_merkle_proof:*/ true)
     }
 
-    fn signed_message(&self) -> &[u8] {
+    fn signed_data(&self) -> &[u8] {
         self.merkle_root().map(AsRef::as_ref).unwrap_or_default()
     }
 }
@@ -553,7 +551,7 @@ impl ShredTrait for ShredCode {
         self.sanitize(/*verify_merkle_proof:*/ true)
     }
 
-    fn signed_message(&self) -> &[u8] {
+    fn signed_data(&self) -> &[u8] {
         self.merkle_root().map(AsRef::as_ref).unwrap_or_default()
     }
 }
@@ -640,7 +638,6 @@ where
     })?
 }
 
-#[cfg(test)]
 fn get_merkle_root(
     shred: &[u8],
     proof_size: u8,
@@ -1347,7 +1344,7 @@ mod test {
             let merkle_branch = make_merkle_branch(index, num_shreds, &tree).unwrap();
             assert_eq!(merkle_branch.proof.len(), usize::from(proof_size));
             shred.set_merkle_branch(&merkle_branch).unwrap();
-            let signature = keypair.sign_message(shred.signed_message());
+            let signature = keypair.sign_message(shred.signed_data());
             shred.set_signature(signature);
             assert!(shred.verify(&keypair.pubkey()));
             assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(()));
@@ -1479,8 +1476,9 @@ mod test {
                 .collect::<Vec<_>>()
         );
         // Assert that shreds sanitize and verify.
+        let pubkey = keypair.pubkey();
         for shred in &shreds {
-            assert!(shred.verify(&keypair.pubkey()));
+            assert!(shred.verify(&pubkey));
             assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(()));
             let ShredCommonHeader {
                 signature,
@@ -1505,9 +1503,11 @@ mod test {
             assert_eq!(shred::layout::get_version(shred), Some(version));
             assert_eq!(shred::layout::get_shred_id(shred), Some(key));
             assert_eq!(shred::layout::get_merkle_root(shred), merkle_root);
-            let slice = shred::layout::get_signed_message_range(shred).unwrap();
-            let message = shred.get(slice).unwrap();
-            assert!(signature.verify(keypair.pubkey().as_ref(), message));
+            let offsets = shred::layout::get_signed_data_offsets(shred).unwrap();
+            let data = shred.get(offsets).unwrap();
+            assert!(signature.verify(pubkey.as_ref(), data));
+            let data = shred::layout::get_signed_data(shred).unwrap();
+            assert!(signature.verify(pubkey.as_ref(), data.as_ref()));
         }
         // Verify common, data and coding headers.
         let mut num_data_shreds = 0;

ledger/src/shred/shred_code.rs

@@ -39,7 +39,7 @@ impl ShredCode {
     dispatch!(pub(super) fn payload(&self) -> &Vec<u8>);
     dispatch!(pub(super) fn sanitize(&self) -> Result<(), Error>);
     dispatch!(pub(super) fn set_signature(&mut self, signature: Signature));
-    dispatch!(pub(super) fn signed_message(&self) -> &[u8]);
+    dispatch!(pub(super) fn signed_data(&self) -> &[u8]);
 
     // Only for tests.
     dispatch!(pub(super) fn set_index(&mut self, index: u32));

ledger/src/shred/shred_data.rs

@@ -29,7 +29,7 @@ impl ShredData {
     dispatch!(pub(super) fn payload(&self) -> &Vec<u8>);
     dispatch!(pub(super) fn sanitize(&self) -> Result<(), Error>);
     dispatch!(pub(super) fn set_signature(&mut self, signature: Signature));
-    dispatch!(pub(super) fn signed_message(&self) -> &[u8]);
+    dispatch!(pub(super) fn signed_data(&self) -> &[u8]);
 
     // Only for tests.
     dispatch!(pub(super) fn set_index(&mut self, index: u32));

ledger/src/shred/traits.rs

@@ -27,7 +27,7 @@ pub(super) trait Shred: Sized {
     fn erasure_shard_as_slice(&self) -> Result<&[u8], Error>;
 
     // Portion of the payload which is signed.
-    fn signed_message(&self) -> &[u8];
+    fn signed_data(&self) -> &[u8];
 
     // Only for tests.
     fn set_index(&mut self, index: u32);