Skip to content

Commit

Permalink
embeds versioning into shred binary
Browse files Browse the repository at this point in the history 
In preparation of
solana-labs#25237
which adds a new shred variant with merkle tree branches, the commit
embeds versioning into shred binary by encoding a new ShredVariant type
at byte 65 of payload replacing previously ShredType at this offset.
    enum ShredVariant {
        LegacyCode, // 0b0101_1010
        LegacyData, // 0b0101_1010
    }

* 0b0101_1010 indicates a legacy coding shred, which is also equal to
  ShredType::Code for backward compatibility.
* 0b1010_0101 indicates a legacy data shred, which is also equal to
  ShredType::Data for backward compatibility.

Following commits will add merkle variants to this type:
    enum ShredVariant {
        LegacyCode, // 0b0101_1010
        LegacyData, // 0b1010_0101
        MerkleCode(/*proof_size:*/ u8), // 0b0100_????
        MerkleData(/*proof_size:*/ u8), // 0b1000_????
    }
  • Loading branch information
@behzadnouri
behzadnouri committed May 30, 2022
1 parent 1727ca4 commit 0d0fadc
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 27 deletions.
128 changes: 108 additions & 20 deletions ledger/src/shred.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ const SIZE_OF_COMMON_SHRED_HEADER: usize = 83;
const SIZE_OF_DATA_SHRED_HEADER: usize = 5;
const SIZE_OF_CODING_SHRED_HEADER: usize = 6;
const SIZE_OF_SIGNATURE: usize = 64;
const SIZE_OF_SHRED_TYPE: usize = 1;
const SIZE_OF_SHRED_VARIANT: usize = 1;
const SIZE_OF_SHRED_SLOT: usize = 8;
const SIZE_OF_SHRED_INDEX: usize = 4;
pub const SIZE_OF_NONCE: usize = 4;
Expand All @@ -108,8 +108,8 @@ pub const SIZE_OF_DATA_SHRED_PAYLOAD: usize = PACKET_DATA_SIZE
const_assert_eq!(SHRED_DATA_OFFSET, 88);
const SHRED_DATA_OFFSET: usize = SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER;

const OFFSET_OF_SHRED_TYPE: usize = SIZE_OF_SIGNATURE;
const OFFSET_OF_SHRED_SLOT: usize = SIZE_OF_SIGNATURE + SIZE_OF_SHRED_TYPE;
const OFFSET_OF_SHRED_VARIANT: usize = SIZE_OF_SIGNATURE;
const OFFSET_OF_SHRED_SLOT: usize = SIZE_OF_SIGNATURE + SIZE_OF_SHRED_VARIANT;
const OFFSET_OF_SHRED_INDEX: usize = OFFSET_OF_SHRED_SLOT + SIZE_OF_SHRED_SLOT;
const_assert_eq!(SHRED_PAYLOAD_SIZE, 1228);
const SHRED_PAYLOAD_SIZE: usize = PACKET_DATA_SIZE - SIZE_OF_NONCE;
Expand Down Expand Up @@ -151,6 +151,8 @@ pub enum Error {
InvalidShredFlags(u8),
#[error("Invalid shred type")]
InvalidShredType,
#[error("Invalid shred variant")]
InvalidShredVariant,
}

#[repr(u8)]
Expand All @@ -174,11 +176,18 @@ pub enum ShredType {
Code = 0b0101_1010,
}

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Deserialize, Serialize)]
#[serde(into = "u8", try_from = "u8")]
enum ShredVariant {
LegacyCode, // 0b0101_1010
LegacyData, // 0b1010_0101
}

/// A common header that is present in data and code shred headers
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
struct ShredCommonHeader {
signature: Signature,
shred_type: ShredType,
shred_variant: ShredVariant,
slot: Slot,
index: u32,
version: u16,
Expand Down Expand Up @@ -317,9 +326,15 @@ impl Shred {
}

pub fn new_from_serialized_shred(shred: Vec<u8>) -> Result<Self, Error> {
Ok(match layout::get_shred_type(&shred)? {
ShredType::Code => Self::from(ShredCode::from_payload(shred)?),
ShredType::Data => Self::from(ShredData::from_payload(shred)?),
Ok(match layout::get_shred_variant(&shred)? {
ShredVariant::LegacyCode => {
let shred = legacy::ShredCode::from_payload(shred)?;
Self::from(shred)
}
ShredVariant::LegacyData => {
let shred = legacy::ShredData::from_payload(shred)?;
Self::from(shred)
}
})
}

Expand Down Expand Up @@ -438,7 +453,7 @@ impl Shred {

#[inline]
pub fn shred_type(&self) -> ShredType {
self.common_header().shred_type
ShredType::from(self.common_header().shred_variant)
}

pub fn is_data(&self) -> bool {
Expand Down Expand Up @@ -531,14 +546,17 @@ pub mod layout {
0..SIZE_OF_SIGNATURE
}

pub(super) fn get_shred_variant(shred: &[u8]) -> Result<ShredVariant, Error> {
let shred_variant = match shred.get(OFFSET_OF_SHRED_VARIANT) {
None => return Err(Error::InvalidPayloadSize(shred.len())),
Some(shred_variant) => *shred_variant,
};
ShredVariant::try_from(shred_variant).map_err(|_| Error::InvalidShredVariant)
}

pub(super) fn get_shred_type(shred: &[u8]) -> Result<ShredType, Error> {
match shred.get(OFFSET_OF_SHRED_TYPE) {
None => Err(Error::InvalidPayloadSize(shred.len())),
Some(shred_type) => match ShredType::try_from(*shred_type) {
Err(_) => Err(Error::InvalidShredType),
Ok(shred_type) => Ok(shred_type),
},
}
let shred_variant = get_shred_variant(shred)?;
Ok(ShredType::from(shred_variant))
}

pub fn get_slot(shred: &[u8]) -> Option<Slot> {
Expand Down Expand Up @@ -585,6 +603,38 @@ impl From<ShredData> for Shred {
}
}

impl From<ShredVariant> for ShredType {
#[inline]
fn from(shred_variant: ShredVariant) -> Self {
match shred_variant {
ShredVariant::LegacyCode => ShredType::Code,
ShredVariant::LegacyData => ShredType::Data,
}
}
}

impl From<ShredVariant> for u8 {
fn from(shred_variant: ShredVariant) -> u8 {
match shred_variant {
ShredVariant::LegacyCode => u8::from(ShredType::Code),
ShredVariant::LegacyData => u8::from(ShredType::Data),
}
}
}

impl TryFrom<u8> for ShredVariant {
type Error = Error;
fn try_from(shred_variant: u8) -> Result<Self, Self::Error> {
if shred_variant == u8::from(ShredType::Code) {
Ok(ShredVariant::LegacyCode)
} else if shred_variant == u8::from(ShredType::Data) {
Ok(ShredVariant::LegacyData)
} else {
Err(Error::InvalidShredVariant)
}
}
}

// Get slot, index, and type from a packet with partial deserialize
pub fn get_shred_slot_index_type(
packet: &Packet,
Expand Down Expand Up @@ -704,7 +754,7 @@ mod tests {
fn test_shred_constants() {
let common_header = ShredCommonHeader {
signature: Signature::default(),
shred_type: ShredType::Code,
shred_variant: ShredVariant::LegacyCode,
slot: Slot::MAX,
index: u32::MAX,
version: u16::MAX,
Expand Down Expand Up @@ -745,8 +795,8 @@ mod tests {
bincode::serialized_size(&Signature::default()).unwrap() as usize
);
assert_eq!(
SIZE_OF_SHRED_TYPE,
bincode::serialized_size(&ShredType::Code).unwrap() as usize
SIZE_OF_SHRED_VARIANT,
bincode::serialized_size(&ShredVariant::LegacyCode).unwrap() as usize
);
assert_eq!(
SIZE_OF_SHRED_SLOT,
Expand Down Expand Up @@ -814,7 +864,7 @@ mod tests {
assert_eq!(Some((1, 3, ShredType::Data)), ret);
assert_eq!(stats, ShredFetchStats::default());

packet.meta.size = OFFSET_OF_SHRED_TYPE;
packet.meta.size = OFFSET_OF_SHRED_VARIANT;
assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
assert_eq!(stats.index_overrun, 1);

Expand Down Expand Up @@ -878,7 +928,7 @@ mod tests {
200, // version
);
shred.copy_to_packet(&mut packet);
packet.buffer_mut()[OFFSET_OF_SHRED_TYPE] = u8::MAX;
packet.buffer_mut()[OFFSET_OF_SHRED_VARIANT] = u8::MAX;

assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
assert_eq!(1, stats.bad_shred_type);
Expand All @@ -894,6 +944,7 @@ mod tests {
assert_matches!(bincode::deserialize::<ShredType>(&[1u8]), Err(_));
// data shred
assert_eq!(ShredType::Data as u8, 0b1010_0101);
assert_eq!(u8::from(ShredType::Data), 0b1010_0101);
assert_eq!(ShredType::try_from(0b1010_0101), Ok(ShredType::Data));
let buf = bincode::serialize(&ShredType::Data).unwrap();
assert_eq!(buf, vec![0b1010_0101]);
Expand All @@ -903,6 +954,7 @@ mod tests {
);
// coding shred
assert_eq!(ShredType::Code as u8, 0b0101_1010);
assert_eq!(u8::from(ShredType::Code), 0b0101_1010);
assert_eq!(ShredType::try_from(0b0101_1010), Ok(ShredType::Code));
let buf = bincode::serialize(&ShredType::Code).unwrap();
assert_eq!(buf, vec![0b0101_1010]);
Expand All @@ -912,6 +964,42 @@ mod tests {
);
}

#[test]
fn test_shred_variant_compat() {
assert_matches!(ShredVariant::try_from(0u8), Err(_));
assert_matches!(ShredVariant::try_from(1u8), Err(_));
assert_matches!(ShredVariant::try_from(0b0101_0000), Err(_));
assert_matches!(ShredVariant::try_from(0b1010_0000), Err(_));
assert_matches!(bincode::deserialize::<ShredVariant>(&[0b0101_0000]), Err(_));
assert_matches!(bincode::deserialize::<ShredVariant>(&[0b1010_0000]), Err(_));
// Legacy coding shred.
assert_eq!(u8::from(ShredVariant::LegacyCode), 0b0101_1010);
assert_eq!(ShredType::from(ShredVariant::LegacyCode), ShredType::Code);
assert_matches!(
ShredVariant::try_from(0b0101_1010),
Ok(ShredVariant::LegacyCode)
);
let buf = bincode::serialize(&ShredVariant::LegacyCode).unwrap();
assert_eq!(buf, vec![0b0101_1010]);
assert_matches!(
bincode::deserialize::<ShredVariant>(&[0b0101_1010]),
Ok(ShredVariant::LegacyCode)
);
// Legacy data shred.
assert_eq!(u8::from(ShredVariant::LegacyData), 0b1010_0101);
assert_eq!(ShredType::from(ShredVariant::LegacyData), ShredType::Data);
assert_matches!(
ShredVariant::try_from(0b1010_0101),
Ok(ShredVariant::LegacyData)
);
let buf = bincode::serialize(&ShredVariant::LegacyData).unwrap();
assert_eq!(buf, vec![0b1010_0101]);
assert_matches!(
bincode::deserialize::<ShredVariant>(&[0b1010_0101]),
Ok(ShredVariant::LegacyData)
);
}

#[test]
fn test_serde_compat_shred_data() {
const SEED: &str = "6qG9NGWEtoTugS4Zgs46u8zTccEJuRHtrNMiUayLHCxt";
Expand Down
14 changes: 7 additions & 7 deletions ledger/src/shred/legacy.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use {
crate::shred::{
traits::{Shred, ShredCode as _, ShredData as _},
CodingShredHeader, DataShredHeader, Error, ShredCommonHeader, ShredFlags, ShredType,
CodingShredHeader, DataShredHeader, Error, ShredCommonHeader, ShredFlags, ShredVariant,
MAX_DATA_SHREDS_PER_FEC_BLOCK, MAX_DATA_SHREDS_PER_SLOT, SHRED_DATA_OFFSET,
SHRED_PAYLOAD_SIZE, SIZE_OF_CODING_SHRED_HEADERS, SIZE_OF_COMMON_SHRED_HEADER,
SIZE_OF_DATA_SHRED_HEADER, SIZE_OF_DATA_SHRED_PAYLOAD, SIZE_OF_SIGNATURE,
Expand Down Expand Up @@ -78,8 +78,8 @@ impl Shred for ShredData {
fn from_payload(mut payload: Vec<u8>) -> Result<Self, Error> {
let mut cursor = Cursor::new(&payload[..]);
let common_header: ShredCommonHeader = deserialize_from_with_limit(&mut cursor)?;
if common_header.shred_type != ShredType::Data {
return Err(Error::InvalidShredType);
if common_header.shred_variant != ShredVariant::LegacyData {
return Err(Error::InvalidShredVariant);
}
let data_header = deserialize_from_with_limit(&mut cursor)?;
// see: https://github.com/solana-labs/solana/pull/16602
Expand Down Expand Up @@ -163,8 +163,8 @@ impl Shred for ShredCode {
fn from_payload(mut payload: Vec<u8>) -> Result<Self, Error> {
let mut cursor = Cursor::new(&payload[..]);
let common_header: ShredCommonHeader = deserialize_from_with_limit(&mut cursor)?;
if common_header.shred_type != ShredType::Code {
return Err(Error::InvalidShredType);
if common_header.shred_variant != ShredVariant::LegacyCode {
return Err(Error::InvalidShredVariant);
}
let coding_header = deserialize_from_with_limit(&mut cursor)?;
// see: https://github.com/solana-labs/solana/pull/10109
Expand Down Expand Up @@ -295,7 +295,7 @@ impl ShredData {
let mut payload = vec![0; SHRED_PAYLOAD_SIZE];
let common_header = ShredCommonHeader {
signature: Signature::default(),
shred_type: ShredType::Data,
shred_variant: ShredVariant::LegacyData,
slot,
index,
version,
Expand Down Expand Up @@ -343,7 +343,7 @@ impl ShredCode {
) -> Self {
let common_header = ShredCommonHeader {
signature: Signature::default(),
shred_type: ShredType::Code,
shred_variant: ShredVariant::LegacyCode,
index,
slot,
version,
Expand Down

0 comments on commit 0d0fadc

Please sign in to comment.