Skip to content

Commit

Permalink
adds Shred{Code,Data}::SIZE_OF_HEADERS trait constants (solana-labs#2…
Browse files Browse the repository at this point in the history 
…7144)
  • Loading branch information
@behzadnouri
behzadnouri authored Aug 15, 2022
1 parent f61f63c commit 0e30609
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 35 deletions.
37 changes: 19 additions & 18 deletions ledger/src/shred/legacy.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@ const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, ShredCode::SIZE_OF_PAYLOAD);
const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, 1228);
const_assert_eq!(ShredData::CAPACITY, 1051);

// SIZE_OF_CODING_SHRED_HEADERS bytes at the end of data shreds
// ShredCode::SIZE_OF_HEADERS bytes at the end of data shreds
// is never used and is not part of erasure coding.
const_assert_eq!(SIZE_OF_ERASURE_ENCODED_SLICE, 1139);
pub(super) const SIZE_OF_ERASURE_ENCODED_SLICE: usize =
ShredCode::SIZE_OF_PAYLOAD - SIZE_OF_CODING_SHRED_HEADERS;
ShredCode::SIZE_OF_PAYLOAD - ShredCode::SIZE_OF_HEADERS;

// Layout: {common, data} headers | data | zero padding
// Everything up to SIZE_OF_CODING_SHRED_HEADERS bytes at the end (which is
// part of zero padding) is erasure coded.
// Everything up to ShredCode::SIZE_OF_HEADERS bytes at the end (which is part
// of zero padding) is erasure coded.
// All payload past signature, including the entirety of zero paddings, is
// signed.
#[derive(Clone, Debug, Eq, PartialEq)]
Expand All @@ -52,6 +52,7 @@ impl Shred for ShredData {
// Legacy data shreds are always zero padded and
// the same size as coding shreds.
const SIZE_OF_PAYLOAD: usize = shred_code::ShredCode::SIZE_OF_PAYLOAD;
const SIZE_OF_HEADERS: usize = SIZE_OF_DATA_SHRED_HEADERS;

fn from_payload(mut payload: Vec<u8>) -> Result<Self, Error> {
let mut cursor = Cursor::new(&payload[..]);
Expand All @@ -64,7 +65,7 @@ impl Shred for ShredData {
// Repair packets have nonce at the end of packet payload; see:
// https://github.com/solana-labs/solana/pull/10109
// https://github.com/solana-labs/solana/pull/16602
if payload.len() < SIZE_OF_DATA_SHRED_HEADERS {
if payload.len() < Self::SIZE_OF_HEADERS {
return Err(Error::InvalidPayloadSize(payload.len()));
}
payload.resize(Self::SIZE_OF_PAYLOAD, 0u8);
Expand Down Expand Up @@ -116,6 +117,7 @@ impl Shred for ShredData {
impl Shred for ShredCode {
impl_shred_common!();
const SIZE_OF_PAYLOAD: usize = shred_code::ShredCode::SIZE_OF_PAYLOAD;
const SIZE_OF_HEADERS: usize = SIZE_OF_CODING_SHRED_HEADERS;

fn from_payload(mut payload: Vec<u8>) -> Result<Self, Error> {
let mut cursor = Cursor::new(&payload[..]);
Expand Down Expand Up @@ -147,18 +149,17 @@ impl Shred for ShredCode {
return Err(Error::InvalidPayloadSize(self.payload.len()));
}
let mut shard = self.payload;
// SIZE_OF_CODING_SHRED_HEADERS bytes at the beginning of the
// coding shreds contains the header and is not part of erasure
// coding.
shard.drain(..SIZE_OF_CODING_SHRED_HEADERS);
// ShredCode::SIZE_OF_HEADERS bytes at the beginning of the coding
// shreds contains the header and is not part of erasure coding.
shard.drain(..Self::SIZE_OF_HEADERS);
Ok(shard)
}

fn erasure_shard_as_slice(&self) -> Result<&[u8], Error> {
if self.payload.len() != Self::SIZE_OF_PAYLOAD {
return Err(Error::InvalidPayloadSize(self.payload.len()));
}
Ok(&self.payload[SIZE_OF_CODING_SHRED_HEADERS..])
Ok(&self.payload[Self::SIZE_OF_HEADERS..])
}

fn sanitize(&self) -> Result<(), Error> {
Expand All @@ -185,15 +186,15 @@ impl ShredDataTrait for ShredData {
let size = usize::from(self.data_header.size);
#[allow(clippy::manual_range_contains)]
if size > self.payload.len()
|| size < SIZE_OF_DATA_SHRED_HEADERS
|| size > SIZE_OF_DATA_SHRED_HEADERS + Self::CAPACITY
|| size < Self::SIZE_OF_HEADERS
|| size > Self::SIZE_OF_HEADERS + Self::CAPACITY
{
return Err(Error::InvalidDataSize {
size: self.data_header.size,
payload: self.payload.len(),
});
}
Ok(&self.payload[SIZE_OF_DATA_SHRED_HEADERS..size])
Ok(&self.payload[Self::SIZE_OF_HEADERS..size])
}

// Only for tests.
Expand All @@ -214,7 +215,7 @@ impl ShredCodeTrait for ShredCode {
impl ShredData {
// Maximum size of ledger data that can be embedded in a data-shred.
pub(super) const CAPACITY: usize =
Self::SIZE_OF_PAYLOAD - SIZE_OF_DATA_SHRED_HEADERS - SIZE_OF_CODING_SHRED_HEADERS;
Self::SIZE_OF_PAYLOAD - Self::SIZE_OF_HEADERS - ShredCode::SIZE_OF_HEADERS;

pub(super) fn new_from_data(
slot: Slot,
Expand All @@ -235,7 +236,7 @@ impl ShredData {
version,
fec_set_index,
};
let size = (data.len() + SIZE_OF_DATA_SHRED_HEADERS) as u16;
let size = (data.len() + Self::SIZE_OF_HEADERS) as u16;
let flags = flags
| unsafe {
ShredFlags::from_bits_unchecked(
Expand All @@ -254,7 +255,7 @@ impl ShredData {
bincode::serialize_into(&mut cursor, &data_header).unwrap();
// TODO: Need to check if data is too large!
let offset = cursor.position() as usize;
debug_assert_eq!(offset, SIZE_OF_DATA_SHRED_HEADERS);
debug_assert_eq!(offset, Self::SIZE_OF_HEADERS);
payload[offset..offset + data.len()].copy_from_slice(data);
Self {
common_header,
Expand All @@ -271,7 +272,7 @@ impl ShredData {

pub(super) fn resize_stored_shred(mut shred: Vec<u8>) -> Result<Vec<u8>, Error> {
// Old shreds might have been extra zero padded.
if !(SIZE_OF_DATA_SHRED_HEADERS..=ShredCode::SIZE_OF_PAYLOAD).contains(&shred.len()) {
if !(Self::SIZE_OF_HEADERS..=Self::SIZE_OF_PAYLOAD).contains(&shred.len()) {
return Err(Error::InvalidPayloadSize(shred.len()));
}
shred.resize(Self::SIZE_OF_PAYLOAD, 0u8);
Expand Down Expand Up @@ -310,7 +311,7 @@ impl ShredCode {
// Tests may have an empty parity_shard.
if !parity_shard.is_empty() {
let offset = cursor.position() as usize;
debug_assert_eq!(offset, SIZE_OF_CODING_SHRED_HEADERS);
debug_assert_eq!(offset, Self::SIZE_OF_HEADERS);
payload[offset..].copy_from_slice(parity_shard);
}
Self {
Expand Down
36 changes: 19 additions & 17 deletions ledger/src/shred/merkle.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,12 @@ impl ShredData {
// Maximum size of ledger data that can be embedded in a data-shred.
// Also equal to:
// ShredCode::size_of_erasure_encoded_slice(proof_size).unwrap()
// - SIZE_OF_DATA_SHRED_HEADERS
// - ShredData::SIZE_OF_HEADERS
// + SIZE_OF_SIGNATURE
pub(super) fn capacity(proof_size: u8) -> Result<usize, Error> {
Self::SIZE_OF_PAYLOAD
.checked_sub(
SIZE_OF_DATA_SHRED_HEADERS
Self::SIZE_OF_HEADERS
+ SIZE_OF_MERKLE_ROOT
+ usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY,
)
Expand All @@ -90,7 +90,7 @@ impl ShredData {

pub(super) fn get_signed_message_range(proof_size: u8) -> Option<Range<usize>> {
let data_buffer_size = Self::capacity(proof_size).ok()?;
let offset = SIZE_OF_DATA_SHRED_HEADERS + data_buffer_size;
let offset = Self::SIZE_OF_HEADERS + data_buffer_size;
Some(offset..offset + SIZE_OF_MERKLE_ROOT)
}

Expand Down Expand Up @@ -121,7 +121,7 @@ impl ShredCode {
// generated. Coding shred headers cannot be erasure coded either.
Self::SIZE_OF_PAYLOAD
.checked_sub(
SIZE_OF_CODING_SHRED_HEADERS
Self::SIZE_OF_HEADERS
+ SIZE_OF_MERKLE_ROOT
+ SIZE_OF_MERKLE_PROOF_ENTRY * usize::from(proof_size),
)
Expand All @@ -133,7 +133,7 @@ impl ShredCode {
let shard_size = Self::size_of_erasure_encoded_slice(proof_size)?;
let chunk = self
.payload
.get(SIZE_OF_SIGNATURE..SIZE_OF_CODING_SHRED_HEADERS + shard_size)
.get(SIZE_OF_SIGNATURE..Self::SIZE_OF_HEADERS + shard_size)
.ok_or(Error::InvalidPayloadSize(self.payload.len()))?;
Ok(hashv(&[MERKLE_HASH_PREFIX_LEAF, chunk]))
}
Expand All @@ -146,7 +146,7 @@ impl ShredCode {

pub(super) fn get_signed_message_range(proof_size: u8) -> Option<Range<usize>> {
let offset =
SIZE_OF_CODING_SHRED_HEADERS + Self::size_of_erasure_encoded_slice(proof_size).ok()?;
Self::SIZE_OF_HEADERS + Self::size_of_erasure_encoded_slice(proof_size).ok()?;
Some(offset..offset + SIZE_OF_MERKLE_ROOT)
}

Expand All @@ -161,12 +161,13 @@ impl Shred for ShredData {
impl_shred_common!();

// Also equal to:
// SIZE_OF_DATA_SHRED_HEADERS
// ShredData::SIZE_OF_HEADERS
// + ShredData::capacity(proof_size).unwrap()
// + SIZE_OF_MERKLE_ROOT
// + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY
const SIZE_OF_PAYLOAD: usize =
ShredCode::SIZE_OF_PAYLOAD - SIZE_OF_CODING_SHRED_HEADERS + SIZE_OF_SIGNATURE;
ShredCode::SIZE_OF_PAYLOAD - ShredCode::SIZE_OF_HEADERS + SIZE_OF_SIGNATURE;
const SIZE_OF_HEADERS: usize = SIZE_OF_DATA_SHRED_HEADERS;

fn from_payload(mut payload: Vec<u8>) -> Result<Self, Error> {
if payload.len() < Self::SIZE_OF_PAYLOAD {
Expand Down Expand Up @@ -213,7 +214,7 @@ impl Shred for ShredData {
let proof_size = self.proof_size()?;
let data_buffer_size = Self::capacity(proof_size)?;
let mut shard = self.payload;
shard.truncate(SIZE_OF_DATA_SHRED_HEADERS + data_buffer_size);
shard.truncate(Self::SIZE_OF_HEADERS + data_buffer_size);
shard.drain(0..SIZE_OF_SIGNATURE);
Ok(shard)
}
Expand All @@ -225,7 +226,7 @@ impl Shred for ShredData {
let proof_size = self.proof_size()?;
let data_buffer_size = Self::capacity(proof_size)?;
self.payload
.get(SIZE_OF_SIGNATURE..SIZE_OF_DATA_SHRED_HEADERS + data_buffer_size)
.get(SIZE_OF_SIGNATURE..Self::SIZE_OF_HEADERS + data_buffer_size)
.ok_or(Error::InvalidPayloadSize(self.payload.len()))
}

Expand All @@ -252,6 +253,7 @@ impl Shred for ShredData {
impl Shred for ShredCode {
impl_shred_common!();
const SIZE_OF_PAYLOAD: usize = shred_code::ShredCode::SIZE_OF_PAYLOAD;
const SIZE_OF_HEADERS: usize = SIZE_OF_CODING_SHRED_HEADERS;

fn from_payload(mut payload: Vec<u8>) -> Result<Self, Error> {
let mut cursor = Cursor::new(&payload[..]);
Expand Down Expand Up @@ -296,7 +298,7 @@ impl Shred for ShredCode {
let proof_size = self.proof_size()?;
let shard_size = Self::size_of_erasure_encoded_slice(proof_size)?;
let mut shard = self.payload;
shard.drain(..SIZE_OF_CODING_SHRED_HEADERS);
shard.drain(..Self::SIZE_OF_HEADERS);
shard.truncate(shard_size);
Ok(shard)
}
Expand All @@ -308,7 +310,7 @@ impl Shred for ShredCode {
let proof_size = self.proof_size()?;
let shard_size = Self::size_of_erasure_encoded_slice(proof_size)?;
self.payload
.get(SIZE_OF_CODING_SHRED_HEADERS..SIZE_OF_CODING_SHRED_HEADERS + shard_size)
.get(Self::SIZE_OF_HEADERS..Self::SIZE_OF_HEADERS + shard_size)
.ok_or(Error::InvalidPayloadSize(self.payload.len()))
}

Expand Down Expand Up @@ -343,15 +345,15 @@ impl ShredDataTrait for ShredData {
let data_buffer_size = Self::capacity(proof_size)?;
let size = usize::from(self.data_header.size);
if size > self.payload.len()
|| size < SIZE_OF_DATA_SHRED_HEADERS
|| size > SIZE_OF_DATA_SHRED_HEADERS + data_buffer_size
|| size < Self::SIZE_OF_HEADERS
|| size > Self::SIZE_OF_HEADERS + data_buffer_size
{
return Err(Error::InvalidDataSize {
size: self.data_header.size,
payload: self.payload.len(),
});
}
Ok(&self.payload[SIZE_OF_DATA_SHRED_HEADERS..size])
Ok(&self.payload[Self::SIZE_OF_HEADERS..size])
}

// Only for tests.
Expand Down Expand Up @@ -439,7 +441,7 @@ mod test {

// Total size of a data shred including headers and merkle branch.
fn shred_data_size_of_payload(proof_size: u8) -> usize {
SIZE_OF_DATA_SHRED_HEADERS
ShredData::SIZE_OF_HEADERS
+ ShredData::capacity(proof_size).unwrap()
+ SIZE_OF_MERKLE_ROOT
+ usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY
Expand All @@ -451,7 +453,7 @@ mod test {
// size of erasure encoded header.
fn shred_data_capacity(proof_size: u8) -> usize {
const SIZE_OF_ERASURE_ENCODED_HEADER: usize =
SIZE_OF_DATA_SHRED_HEADERS - SIZE_OF_SIGNATURE;
ShredData::SIZE_OF_HEADERS - SIZE_OF_SIGNATURE;
ShredCode::size_of_erasure_encoded_slice(proof_size).unwrap()
- SIZE_OF_ERASURE_ENCODED_HEADER
}
Expand Down
2 changes: 2 additions & 0 deletions ledger/src/shred/traits.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ pub(super) trait Shred: Sized {
// Total size of payload including headers, merkle
// branches (if any), zero paddings, etc.
const SIZE_OF_PAYLOAD: usize;
// Size of common and code/data headers.
const SIZE_OF_HEADERS: usize;

fn from_payload(shred: Vec<u8>) -> Result<Self, Error>;
fn common_header(&self) -> &ShredCommonHeader;
Expand Down

0 comments on commit 0e30609

Please sign in to comment.