diff --git a/ledger/src/shred.rs b/ledger/src/shred.rs index 950faa3a73288d..73e5759a0b8f99 100644 --- a/ledger/src/shred.rs +++ b/ledger/src/shred.rs @@ -309,7 +309,7 @@ use dispatch; impl Shred { dispatch!(fn common_header(&self) -> &ShredCommonHeader); dispatch!(fn set_signature(&mut self, signature: Signature)); - dispatch!(fn signed_message(&self) -> &[u8]); + dispatch!(fn signed_data(&self) -> &[u8]); // Returns the portion of the shred's payload which is erasure coded. dispatch!(pub(crate) fn erasure_shard(self) -> Result, Error>); @@ -460,7 +460,7 @@ impl Shred { } pub fn sign(&mut self, keypair: &Keypair) { - let signature = keypair.sign_message(self.signed_message()); + let signature = keypair.sign_message(self.signed_data()); self.set_signature(signature); } @@ -508,8 +508,8 @@ impl Shred { #[must_use] pub fn verify(&self, pubkey: &Pubkey) -> bool { - let message = self.signed_message(); - self.signature().verify(pubkey.as_ref(), message) + let data = self.signed_data(); + self.signature().verify(pubkey.as_ref(), data) } // Returns true if the erasure coding of the two shreds mismatch. @@ -538,9 +538,26 @@ impl Shred { // Helper methods to extract pieces of the shred from the payload // without deserializing the entire payload. pub mod layout { + use {super::*, crate::shred::merkle::MerkleRoot, std::ops::Range}; #[cfg(test)] - use crate::shred::merkle::MerkleRoot; - use {super::*, std::ops::Range}; + use { + rand::{seq::SliceRandom, Rng}, + std::collections::HashMap, + }; + + pub(crate) enum SignedData<'a> { + Chunk(&'a [u8]), + MerkleRoot(MerkleRoot), + } + + impl<'a> AsRef<[u8]> for SignedData<'a> { + fn as_ref(&self) -> &[u8] { + match self { + Self::Chunk(chunk) => chunk, + Self::MerkleRoot(root) => root, + } + } + } fn get_shred_size(packet: &Packet) -> Option { let size = packet.data(..)?.len(); @@ -615,18 +632,36 @@ pub mod layout { )) } - // Returns slice range of the shred payload which is signed. - pub(crate) fn get_signed_message_range(shred: &[u8]) -> Option> { - let range = match get_shred_variant(shred).ok()? { - ShredVariant::LegacyCode | ShredVariant::LegacyData => legacy::SIGNED_MESSAGE_RANGE, + pub(crate) fn get_signed_data(shred: &[u8]) -> Option { + let data = match get_shred_variant(shred).ok()? { + ShredVariant::LegacyCode | ShredVariant::LegacyData => { + let chunk = shred.get(self::legacy::SIGNED_MESSAGE_OFFSETS)?; + SignedData::Chunk(chunk) + } + ShredVariant::MerkleCode(proof_size) => { + let merkle_root = self::merkle::ShredCode::get_merkle_root(shred, proof_size)?; + SignedData::MerkleRoot(merkle_root) + } + ShredVariant::MerkleData(proof_size) => { + let merkle_root = self::merkle::ShredData::get_merkle_root(shred, proof_size)?; + SignedData::MerkleRoot(merkle_root) + } + }; + Some(data) + } + + // Returns offsets within the shred payload which is signed. + pub(crate) fn get_signed_data_offsets(shred: &[u8]) -> Option> { + let offsets = match get_shred_variant(shred).ok()? { + ShredVariant::LegacyCode | ShredVariant::LegacyData => legacy::SIGNED_MESSAGE_OFFSETS, ShredVariant::MerkleCode(proof_size) => { - merkle::ShredCode::get_signed_message_range(proof_size)? + merkle::ShredCode::get_signed_data_offsets(proof_size)? } ShredVariant::MerkleData(proof_size) => { - merkle::ShredData::get_signed_message_range(proof_size)? + merkle::ShredData::get_signed_data_offsets(proof_size)? } }; - (range.end <= shred.len()).then(|| range) + (offsets.end <= shred.len()).then(|| offsets) } pub(crate) fn get_reference_tick(shred: &[u8]) -> Result { @@ -652,6 +687,62 @@ pub mod layout { } } } + + // Minimally corrupts the packet so that the signature no longer verifies. + #[cfg(test)] + pub(crate) fn corrupt_packet( + rng: &mut R, + packet: &mut Packet, + keypairs: &HashMap, + ) { + fn modify_packet(rng: &mut R, packet: &mut Packet, offsets: Range) { + let buffer = packet.buffer_mut(); + let byte = buffer[offsets].choose_mut(rng).unwrap(); + *byte = rng.gen::().max(1u8).wrapping_add(*byte); + } + let shred = get_shred(packet).unwrap(); + let merkle_proof_size = match get_shred_variant(shred).unwrap() { + ShredVariant::LegacyCode | ShredVariant::LegacyData => None, + ShredVariant::MerkleCode(proof_size) | ShredVariant::MerkleData(proof_size) => { + Some(proof_size) + } + }; + let coin_flip: bool = rng.gen(); + if coin_flip { + // Corrupt one byte within the signature offsets. + modify_packet(rng, packet, 0..SIGNATURE_BYTES); + } else { + // Corrupt one byte within the signed data offsets. + let size = shred.len(); + let offsets = get_signed_data_offsets(shred).unwrap(); + modify_packet(rng, packet, offsets); + if let Some(proof_size) = merkle_proof_size { + // Also need to corrupt the merkle proof. + // Proof entries are each 20 bytes at the end of shreds. + let offset = usize::from(proof_size) * 20; + modify_packet(rng, packet, size - offset..size); + } + } + // Assert that the signature no longer verifies. + let shred = get_shred(packet).unwrap(); + let slot = get_slot(shred).unwrap(); + let signature = get_signature(shred).unwrap(); + if coin_flip { + let pubkey = keypairs[&slot].pubkey(); + let data = get_signed_data(shred).unwrap(); + assert!(!signature.verify(pubkey.as_ref(), data.as_ref())); + let offsets = get_signed_data_offsets(shred).unwrap(); + assert!(!signature.verify(pubkey.as_ref(), &shred[offsets])); + } else { + // Slot may have been corrupted and no longer mapping to a keypair. + let pubkey = keypairs.get(&slot).map(Keypair::pubkey).unwrap_or_default(); + if let Some(data) = get_signed_data(shred) { + assert!(!signature.verify(pubkey.as_ref(), data.as_ref())); + } + let offsets = get_signed_data_offsets(shred).unwrap_or_default(); + assert!(!signature.verify(pubkey.as_ref(), &shred[offsets])); + } + } } impl From for Shred { diff --git a/ledger/src/shred/legacy.rs b/ledger/src/shred/legacy.rs index 62eb58bc38e164..095ee8691b82e8 100644 --- a/ledger/src/shred/legacy.rs +++ b/ledger/src/shred/legacy.rs @@ -15,7 +15,8 @@ use { // All payload including any zero paddings are signed. // Code and data shreds have the same payload size. -pub(super) const SIGNED_MESSAGE_RANGE: Range = SIZE_OF_SIGNATURE..ShredData::SIZE_OF_PAYLOAD; +pub(super) const SIGNED_MESSAGE_OFFSETS: Range = + SIZE_OF_SIGNATURE..ShredData::SIZE_OF_PAYLOAD; const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, ShredCode::SIZE_OF_PAYLOAD); const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, 1228); const_assert_eq!(ShredData::CAPACITY, 1051); @@ -108,7 +109,7 @@ impl Shred for ShredData { shred_data::sanitize(self) } - fn signed_message(&self) -> &[u8] { + fn signed_data(&self) -> &[u8] { debug_assert_eq!(self.payload.len(), Self::SIZE_OF_PAYLOAD); &self.payload[SIZE_OF_SIGNATURE..] } @@ -170,7 +171,7 @@ impl Shred for ShredCode { shred_code::sanitize(self) } - fn signed_message(&self) -> &[u8] { + fn signed_data(&self) -> &[u8] { debug_assert_eq!(self.payload.len(), Self::SIZE_OF_PAYLOAD); &self.payload[SIZE_OF_SIGNATURE..] } diff --git a/ledger/src/shred/merkle.rs b/ledger/src/shred/merkle.rs index e86dcd0a1745b0..13bc6f861f5e3a 100644 --- a/ledger/src/shred/merkle.rs +++ b/ledger/src/shred/merkle.rs @@ -91,12 +91,12 @@ impl Shred { dispatch!(fn sanitize(&self, verify_merkle_proof: bool) -> Result<(), Error>); dispatch!(fn set_merkle_branch(&mut self, merkle_branch: &MerkleBranch) -> Result<(), Error>); dispatch!(fn set_signature(&mut self, signature: Signature)); - dispatch!(fn signed_message(&self) -> &[u8]); + dispatch!(fn signed_data(&self) -> &[u8]); #[must_use] fn verify(&self, pubkey: &Pubkey) -> bool { - let message = self.signed_message(); - self.signature().verify(pubkey.as_ref(), message) + let data = self.signed_data(); + self.signature().verify(pubkey.as_ref(), data) } fn signature(&self) -> Signature { @@ -147,7 +147,7 @@ impl ShredData { .ok_or(Error::InvalidProofSize(proof_size)) } - pub(super) fn get_signed_message_range(proof_size: u8) -> Option> { + pub(super) fn get_signed_data_offsets(proof_size: u8) -> Option> { let data_buffer_size = Self::capacity(proof_size).ok()?; let offset = Self::SIZE_OF_HEADERS + data_buffer_size; Some(offset..offset + SIZE_OF_MERKLE_ROOT) @@ -247,7 +247,6 @@ impl ShredData { shred_data::sanitize(self) } - #[cfg(test)] pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option { debug_assert_eq!( shred::layout::get_shred_variant(shred).unwrap(), @@ -332,7 +331,7 @@ impl ShredCode { Ok(verify_merkle_proof(index, node, &self.merkle_branch()?)) } - pub(super) fn get_signed_message_range(proof_size: u8) -> Option> { + pub(super) fn get_signed_data_offsets(proof_size: u8) -> Option> { let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size).ok()?; Some(offset..offset + SIZE_OF_MERKLE_ROOT) } @@ -397,7 +396,6 @@ impl ShredCode { shred_code::sanitize(self) } - #[cfg(test)] pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option { debug_assert_eq!( shred::layout::get_shred_variant(shred).unwrap(), @@ -488,7 +486,7 @@ impl ShredTrait for ShredData { self.sanitize(/*verify_merkle_proof:*/ true) } - fn signed_message(&self) -> &[u8] { + fn signed_data(&self) -> &[u8] { self.merkle_root().map(AsRef::as_ref).unwrap_or_default() } } @@ -553,7 +551,7 @@ impl ShredTrait for ShredCode { self.sanitize(/*verify_merkle_proof:*/ true) } - fn signed_message(&self) -> &[u8] { + fn signed_data(&self) -> &[u8] { self.merkle_root().map(AsRef::as_ref).unwrap_or_default() } } @@ -640,7 +638,6 @@ where })? } -#[cfg(test)] fn get_merkle_root( shred: &[u8], proof_size: u8, @@ -1347,7 +1344,7 @@ mod test { let merkle_branch = make_merkle_branch(index, num_shreds, &tree).unwrap(); assert_eq!(merkle_branch.proof.len(), usize::from(proof_size)); shred.set_merkle_branch(&merkle_branch).unwrap(); - let signature = keypair.sign_message(shred.signed_message()); + let signature = keypair.sign_message(shred.signed_data()); shred.set_signature(signature); assert!(shred.verify(&keypair.pubkey())); assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(())); @@ -1479,8 +1476,9 @@ mod test { .collect::>() ); // Assert that shreds sanitize and verify. + let pubkey = keypair.pubkey(); for shred in &shreds { - assert!(shred.verify(&keypair.pubkey())); + assert!(shred.verify(&pubkey)); assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(())); let ShredCommonHeader { signature, @@ -1505,9 +1503,11 @@ mod test { assert_eq!(shred::layout::get_version(shred), Some(version)); assert_eq!(shred::layout::get_shred_id(shred), Some(key)); assert_eq!(shred::layout::get_merkle_root(shred), merkle_root); - let slice = shred::layout::get_signed_message_range(shred).unwrap(); - let message = shred.get(slice).unwrap(); - assert!(signature.verify(keypair.pubkey().as_ref(), message)); + let offsets = shred::layout::get_signed_data_offsets(shred).unwrap(); + let data = shred.get(offsets).unwrap(); + assert!(signature.verify(pubkey.as_ref(), data)); + let data = shred::layout::get_signed_data(shred).unwrap(); + assert!(signature.verify(pubkey.as_ref(), data.as_ref())); } // Verify common, data and coding headers. let mut num_data_shreds = 0; diff --git a/ledger/src/shred/shred_code.rs b/ledger/src/shred/shred_code.rs index 0cafc6c4ada6bd..68c4f04560d89d 100644 --- a/ledger/src/shred/shred_code.rs +++ b/ledger/src/shred/shred_code.rs @@ -39,7 +39,7 @@ impl ShredCode { dispatch!(pub(super) fn payload(&self) -> &Vec); dispatch!(pub(super) fn sanitize(&self) -> Result<(), Error>); dispatch!(pub(super) fn set_signature(&mut self, signature: Signature)); - dispatch!(pub(super) fn signed_message(&self) -> &[u8]); + dispatch!(pub(super) fn signed_data(&self) -> &[u8]); // Only for tests. dispatch!(pub(super) fn set_index(&mut self, index: u32)); diff --git a/ledger/src/shred/shred_data.rs b/ledger/src/shred/shred_data.rs index 73c8ef221645d5..7e1e1e2bb0baaa 100644 --- a/ledger/src/shred/shred_data.rs +++ b/ledger/src/shred/shred_data.rs @@ -29,7 +29,7 @@ impl ShredData { dispatch!(pub(super) fn payload(&self) -> &Vec); dispatch!(pub(super) fn sanitize(&self) -> Result<(), Error>); dispatch!(pub(super) fn set_signature(&mut self, signature: Signature)); - dispatch!(pub(super) fn signed_message(&self) -> &[u8]); + dispatch!(pub(super) fn signed_data(&self) -> &[u8]); // Only for tests. dispatch!(pub(super) fn set_index(&mut self, index: u32)); diff --git a/ledger/src/shred/traits.rs b/ledger/src/shred/traits.rs index 5ae2b955aaa45a..3cb9b01403e1ff 100644 --- a/ledger/src/shred/traits.rs +++ b/ledger/src/shred/traits.rs @@ -27,7 +27,7 @@ pub(super) trait Shred: Sized { fn erasure_shard_as_slice(&self) -> Result<&[u8], Error>; // Portion of the payload which is signed. - fn signed_message(&self) -> &[u8]; + fn signed_data(&self) -> &[u8]; // Only for tests. fn set_index(&mut self, index: u32); diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index 8f1d0e09b5c60b..b350947052592f 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -57,12 +57,11 @@ pub fn verify_shred_cpu( Some(signature) => signature, }; trace!("signature {}", signature); - let message = - match shred::layout::get_signed_message_range(shred).and_then(|slice| shred.get(slice)) { - None => return false, - Some(message) => message, - }; - signature.verify(pubkey, message) + let data = match shred::layout::get_signed_data(shred) { + None => return false, + Some(data) => data, + }; + signature.verify(pubkey, data.as_ref()) } fn verify_shreds_cpu( @@ -182,7 +181,7 @@ fn shred_gpu_offsets( let shred = shred::layout::get_shred(packet); // Signature may verify for an empty message but the packet will be // discarded during deserialization. - let msg = shred.and_then(shred::layout::get_signed_message_range); + let msg = shred.and_then(shred::layout::get_signed_data_offsets); let msg = add_offset(msg.unwrap_or_default(), offset); signature_offsets.push(sig.start as u32); msg_start_offsets.push(msg.start as u32); @@ -262,13 +261,13 @@ pub fn verify_shreds_gpu( fn sign_shred_cpu(keypair: &Keypair, packet: &mut Packet) { let sig = shred::layout::get_signature_range(); let msg = shred::layout::get_shred(packet) - .and_then(shred::layout::get_signed_message_range) + .and_then(shred::layout::get_signed_data) .unwrap(); assert!( packet.meta.size >= sig.end, "packet is not large enough for a signature" ); - let signature = keypair.sign_message(packet.data(msg).unwrap()); + let signature = keypair.sign_message(msg.as_ref()); trace!("signature {:?}", signature); packet.buffer_mut()[sig].copy_from_slice(signature.as_ref()); } @@ -415,7 +414,7 @@ mod tests { solana_sdk::{ hash, hash::Hash, - signature::{Keypair, Signer, SIGNATURE_BYTES}, + signature::{Keypair, Signer}, system_transaction, transaction::Transaction, }, @@ -690,98 +689,42 @@ mod tests { ) } - // Minimally corrupts the packet so that the signature no longer verifies. - fn corrupt_packet(rng: &mut R, packet: &mut Packet, keypairs: &HashMap) { - let coin_flip: bool = rng.gen(); - if coin_flip { - // Corrupt one byte within the signature offsets. - let k = rng.gen_range(0, SIGNATURE_BYTES); - let buffer = packet.buffer_mut(); - buffer[k] = buffer[k].wrapping_add(1u8); - } else { - // Corrupt one byte within the signed message offsets. - let shred = shred::layout::get_shred(packet).unwrap(); - let offsets: Range = shred::layout::get_signed_message_range(shred).unwrap(); - let k = rng.gen_range(offsets.start, offsets.end); - let buffer = packet.buffer_mut(); - buffer[k] = buffer[k].wrapping_add(1u8); - } - // Assert that the signature no longer verifies. - let shred = shred::layout::get_shred(packet).unwrap(); - let slot = shred::layout::get_slot(shred).unwrap(); - let signature = shred::layout::get_signature(shred).unwrap(); - if coin_flip { - let pubkey = keypairs[&slot].pubkey(); - let offsets = shred::layout::get_signed_message_range(shred).unwrap(); - assert!(!signature.verify(pubkey.as_ref(), &shred[offsets])); - } else { - // Slot may have been corrupted and no longer mapping to a keypair. - let pubkey = keypairs.get(&slot).map(Keypair::pubkey).unwrap_or_default(); - let offsets = shred::layout::get_signed_message_range(shred).unwrap_or_default(); - assert!(!signature.verify(pubkey.as_ref(), &shred[offsets])); - }; + fn make_entries(rng: &mut R, num_entries: usize) -> Vec { + let prev_hash = hash::hashv(&[&rng.gen::<[u8; 32]>()]); + let entry = make_entry(rng, &prev_hash); + std::iter::successors(Some(entry), |entry| Some(make_entry(rng, &entry.hash))) + .take(num_entries) + .collect() } - fn make_shreds(rng: &mut R) -> (Vec, HashMap) { + fn make_shreds(rng: &mut R, keypairs: &HashMap) -> Vec { let reed_solomon_cache = ReedSolomonCache::default(); - let entries: Vec<_> = { - let prev_hash = hash::hashv(&[&rng.gen::<[u8; 32]>()]); - let entry = make_entry(rng, &prev_hash); - let num_entries = rng.gen_range(64, 128); - std::iter::successors(Some(entry), |entry| Some(make_entry(rng, &entry.hash))) - .take(num_entries) - .collect() - }; - let mut keypairs = HashMap::::new(); - // Legacy shreds. - let (mut shreds, coding_shreds) = { - let slot = 169_367_809; - let parent_slot = slot - rng.gen::().max(1) as Slot; - keypairs.insert(slot, Keypair::new()); - Shredder::new( - slot, - parent_slot, - rng.gen_range(0, 0x3F), // reference_tick - rng.gen(), // version - ) - .unwrap() - .entries_to_shreds( - &keypairs[&slot], - &entries, - rng.gen(), // is_last_in_slot - rng.gen_range(0, 671), // next_shred_index - rng.gen_range(0, 781), // next_code_index - false, // merkle_variant - &reed_solomon_cache, - &mut ProcessShredsStats::default(), - ) - }; - shreds.extend(coding_shreds); - // Merkle shreds. - let (data_shreds, coding_shreds) = { - let slot = 169_376_655; - let parent_slot = slot - rng.gen::().max(1) as Slot; - keypairs.insert(slot, Keypair::new()); - Shredder::new( - slot, - parent_slot, - rng.gen_range(0, 0x3F), // reference_tick - rng.gen(), // version - ) - .unwrap() - .entries_to_shreds( - &keypairs[&slot], - &entries, - rng.gen(), // is_last_in_slot - rng.gen_range(0, 671), // next_shred_index - rng.gen_range(0, 781), // next_code_index - true, // merkle_variant - &reed_solomon_cache, - &mut ProcessShredsStats::default(), - ) - }; - shreds.extend(data_shreds); - shreds.extend(coding_shreds); + let mut shreds: Vec<_> = keypairs + .iter() + .flat_map(|(&slot, keypair)| { + let parent_slot = slot - rng.gen::().max(1) as Slot; + let num_entries = rng.gen_range(64, 128); + let (data_shreds, coding_shreds) = Shredder::new( + slot, + parent_slot, + rng.gen_range(0, 0x40), // reference_tick + rng.gen(), // version + ) + .unwrap() + .entries_to_shreds( + keypair, + &make_entries(rng, num_entries), + rng.gen(), // is_last_in_slot + rng.gen_range(0, 2671), // next_shred_index + rng.gen_range(0, 2781), // next_code_index + rng.gen(), // merkle_variant, + &reed_solomon_cache, + &mut ProcessShredsStats::default(), + ); + [data_shreds, coding_shreds] + }) + .flatten() + .collect(); shreds.shuffle(rng); // Assert that all shreds verfiy and sanitize. for shred in &shreds { @@ -794,11 +737,13 @@ mod tests { let shred = shred.payload(); let slot = shred::layout::get_slot(shred).unwrap(); let signature = shred::layout::get_signature(shred).unwrap(); - let offsets = shred::layout::get_signed_message_range(shred).unwrap(); + let offsets = shred::layout::get_signed_data_offsets(shred).unwrap(); let pubkey = keypairs[&slot].pubkey(); assert!(signature.verify(pubkey.as_ref(), &shred[offsets])); + let data = shred::layout::get_signed_data(shred).unwrap(); + assert!(signature.verify(pubkey.as_ref(), data.as_ref())); } - (shreds, keypairs) + shreds } fn make_packets(rng: &mut R, shreds: &[Shred]) -> Vec { @@ -827,7 +772,11 @@ mod tests { fn test_verify_shreds_fuzz() { let mut rng = rand::thread_rng(); let recycler_cache = RecyclerCache::default(); - let (shreds, keypairs) = make_shreds(&mut rng); + let keypairs = repeat_with(|| rng.gen_range(169_367_809, 169_906_789)) + .map(|slot| (slot, Keypair::new())) + .take(3) + .collect(); + let shreds = make_shreds(&mut rng, &keypairs); let pubkeys: HashMap = keypairs .iter() .map(|(&slot, keypair)| (slot, keypair.pubkey().to_bytes())) @@ -850,7 +799,7 @@ mod tests { .map(|packet| { let coin_flip: bool = rng.gen(); if !coin_flip { - corrupt_packet(&mut rng, packet, &keypairs); + shred::layout::corrupt_packet(&mut rng, packet, &keypairs); } u8::from(coin_flip) }) @@ -864,7 +813,13 @@ mod tests { fn test_sign_shreds_gpu() { let mut rng = rand::thread_rng(); let recycler_cache = RecyclerCache::default(); - let (shreds, _) = make_shreds(&mut rng); + let shreds = { + let keypairs = repeat_with(|| rng.gen_range(169_367_809, 169_906_789)) + .map(|slot| (slot, Keypair::new())) + .take(3) + .collect(); + make_shreds(&mut rng, &keypairs) + }; let keypair = Keypair::new(); let pubkeys: HashMap = { let pubkey = keypair.pubkey().to_bytes();