Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ledger/blockstore: Use u32 for shred indices. #2452

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

ledger/blockstore: Use u32 for shred indices. #2452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions core/benches/shredder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ fn make_large_unchained_entries(txs_per_entry: u64, num_entries: u64) -> Vec<Ent
.collect()
}

fn make_shreds(num_shreds: usize) -> Vec<Shred> {
fn make_shreds(num_shreds: u32) -> Vec<Shred> {
let txs_per_entry = 128;
let num_entries = max_entries_per_n_shred(
&make_test_entry(txs_per_entry),
2 * num_shreds as u64,
2 * num_shreds,
Some(LEGACY_SHRED_DATA_CAPACITY),
);
let entries = make_large_unchained_entries(txs_per_entry, num_entries);
Expand All @@ -49,15 +49,15 @@ fn make_shreds(num_shreds: usize) -> Vec<Shred> {
&ReedSolomonCache::default(),
&mut ProcessShredsStats::default(),
);
assert!(data_shreds.len() >= num_shreds);
assert!(data_shreds.len() >= num_shreds as usize);
data_shreds
}

#[bench]
fn bench_shredder_ticks(bencher: &mut Bencher) {
let kp = Keypair::new();
let shred_size = LEGACY_SHRED_DATA_CAPACITY;
let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
let num_shreds = u32::try_from(((1000 * 1000) + (shred_size - 1)) / shred_size).unwrap();
// ~1Mb
let num_ticks = max_ticks_per_n_shreds(1, Some(LEGACY_SHRED_DATA_CAPACITY)) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default());
Expand All @@ -83,11 +83,11 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
fn bench_shredder_large_entries(bencher: &mut Bencher) {
let kp = Keypair::new();
let shred_size = LEGACY_SHRED_DATA_CAPACITY;
let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
let num_shreds = u32::try_from(((1000 * 1000) + (shred_size - 1)) / shred_size).unwrap();
let txs_per_entry = 128;
let num_entries = max_entries_per_n_shred(
&make_test_entry(txs_per_entry),
num_shreds as u64,
num_shreds,
Some(shred_size),
);
let entries = make_large_unchained_entries(txs_per_entry, num_entries);
Expand Down Expand Up @@ -115,7 +115,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
let kp = Keypair::new();
let shred_size = LEGACY_SHRED_DATA_CAPACITY;
// ~10Mb
let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
let num_shreds = u32::try_from(((10000 * 1000) + (shred_size - 1)) / shred_size).unwrap();
let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default());
let shredder = Shredder::new(1, 0, 0, 0).unwrap();
Expand Down Expand Up @@ -156,7 +156,7 @@ fn bench_shredder_coding(bencher: &mut Bencher) {
let reed_solomon_cache = ReedSolomonCache::default();
bencher.iter(|| {
Shredder::generate_coding_shreds(
&data_shreds[..symbol_count],
&data_shreds[..symbol_count as usize],
0, // next_code_index
&reed_solomon_cache,
)
Expand All @@ -170,7 +170,7 @@ fn bench_shredder_decoding(bencher: &mut Bencher) {
let data_shreds = make_shreds(symbol_count);
let reed_solomon_cache = ReedSolomonCache::default();
let coding_shreds = Shredder::generate_coding_shreds(
&data_shreds[..symbol_count],
&data_shreds[..symbol_count as usize],
0, // next_code_index
&reed_solomon_cache,
);
Expand Down
2 changes: 1 addition & 1 deletion core/src/repair/repair_response.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use {
pub fn repair_response_packet(
blockstore: &Blockstore,
slot: Slot,
shred_index: u64,
shred_index: u32,
dest: &SocketAddr,
nonce: Nonce,
) -> Option<Packet> {
Expand Down
6 changes: 5 additions & 1 deletion core/src/repair/repair_service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,11 @@ impl RepairService {
if let Some(reference_tick) = slot_meta
.received
.checked_sub(1)
.and_then(|index| blockstore.get_data_shred(slot, index).ok()?)
.and_then(|index| {
blockstore
.get_data_shred(slot, u32::try_from(index).ok()?)
.ok()?
})
.and_then(|shred| shred::layout::get_reference_tick(&shred).ok())
.map(u64::from)
{
Expand Down
32 changes: 20 additions & 12 deletions core/src/repair/serve_repair.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,12 @@ pub enum ShredRepairType {
/// Requesting `MAX_ORPHAN_REPAIR_RESPONSES ` parent shreds
Orphan(Slot),
/// Requesting any shred with index greater than or equal to the particular index
// TODO As the value is a shred index, it should use `u32`. But as changing the type would
// affect the serialized representation, we would need to introduce a conversion.
HighestShred(Slot, u64),
/// Requesting the missing shred at a particular index
// TODO As the value is a shred index, it should use `u32`. But as changing the type would
// affect the serialized representation, we would need to introduce a conversion.
Shred(Slot, u64),
}

Expand Down Expand Up @@ -221,6 +225,7 @@ impl RepairRequestHeader {
pub(crate) type Ping = ping_pong::Ping<[u8; REPAIR_PING_TOKEN_SIZE]>;

/// Window protocol messages
// TODO All `u64`s in the branches of this enum are shred indices, and they should be used `u32`.
#[cfg_attr(
feature = "frozen-abi",
derive(AbiEnumVisitor, AbiExample),
Expand Down Expand Up @@ -437,7 +442,7 @@ impl ServeRepair {
from_addr,
blockstore,
*slot,
*shred_index,
u32::try_from(*shred_index).expect("All shred indices fit into u32"),
*nonce,
);
if batch.is_none() {
Expand All @@ -457,7 +462,7 @@ impl ServeRepair {
from_addr,
blockstore,
*slot,
*highest_index,
u32::try_from(*highest_index).expect("All shred indices fit into u32"),
*nonce,
),
"HighestWindowIndexWithNonce",
Expand Down Expand Up @@ -1290,7 +1295,7 @@ impl ServeRepair {
from_addr: &SocketAddr,
blockstore: &Blockstore,
slot: Slot,
shred_index: u64,
shred_index: u32,
nonce: Nonce,
) -> Option<PacketBatch> {
// Try to find the requested index in one of the slots
Expand All @@ -1313,17 +1318,17 @@ impl ServeRepair {
from_addr: &SocketAddr,
blockstore: &Blockstore,
slot: Slot,
highest_index: u64,
highest_index: u32,
nonce: Nonce,
) -> Option<PacketBatch> {
// Try to find the requested index in one of the slots
let meta = blockstore.meta(slot).ok()??;
if meta.received > highest_index {
if meta.received > highest_index.into() {
// meta.received must be at least 1 by this point
let packet = repair_response::repair_response_packet(
blockstore,
slot,
meta.received - 1,
u32::try_from(meta.received - 1).expect("All shred indices fit into u32"),
from_addr,
nonce,
)?;
Expand Down Expand Up @@ -1354,7 +1359,8 @@ impl ServeRepair {
repair_response::repair_response_packet(
blockstore,
meta.slot,
meta.received.checked_sub(1u64)?,
u32::try_from(meta.received.checked_sub(1u64)?)
.expect("All shred indices fit into u32"),
from_addr,
nonce,
)
Expand Down Expand Up @@ -1909,7 +1915,7 @@ mod tests {
nonce,
)
.expect("packets");
let request = ShredRepairType::HighestShred(slot, index);
let request = ShredRepairType::HighestShred(slot, index.into());
verify_responses(&request, rv.iter());

let rv: Vec<Shred> = rv
Expand All @@ -1920,8 +1926,10 @@ mod tests {
})
.collect();
assert!(!rv.is_empty());
let index = blockstore.meta(slot).unwrap().unwrap().received - 1;
assert_eq!(rv[0].index(), index as u32);
let index: u32 = (blockstore.meta(slot).unwrap().unwrap().received - 1)
.try_into()
.unwrap();
assert_eq!(rv[0].index(), index);
assert_eq!(rv[0].slot(), slot);

let rv = ServeRepair::run_highest_window_request(
Expand Down Expand Up @@ -1971,7 +1979,7 @@ mod tests {
nonce,
)
.expect("packets");
let request = ShredRepairType::Shred(slot, index);
let request = ShredRepairType::Shred(slot, index.into());
verify_responses(&request, rv.iter());
let rv: Vec<Shred> = rv
.into_iter()
Expand Down Expand Up @@ -2163,7 +2171,7 @@ mod tests {
repair_response::repair_response_packet(
&blockstore,
slot,
index,
index.try_into().unwrap(),
&socketaddr_any!(),
nonce,
)
Expand Down
12 changes: 6 additions & 6 deletions ledger/benches/blockstore.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ fn bench_write_shreds(bench: &mut Bencher, entries: Vec<Entry>, ledger_path: &Pa
// Insert some shreds into the ledger in preparation for read benchmarks
fn setup_read_bench(
blockstore: &Blockstore,
num_small_shreds: u64,
num_large_shreds: u64,
num_small_shreds: u32,
num_large_shreds: u32,
slot: Slot,
) {
// Make some big and small entries
let entries = create_ticks(
num_large_shreds * 4 + num_small_shreds * 2,
(num_large_shreds * 4 + num_small_shreds * 2).into(),
0,
Hash::default(),
);
Expand Down Expand Up @@ -117,12 +117,12 @@ fn bench_read_random(bench: &mut Bencher) {
// Generate a num_reads sized random sample of indexes in range [0, total_shreds - 1],
// simulating random reads
let mut rng = rand::thread_rng();
let indexes: Vec<usize> = (0..num_reads)
.map(|_| rng.gen_range(0..total_shreds) as usize)
let indexes: Vec<u32> = (0..num_reads)
.map(|_| rng.gen_range(0..total_shreds))
.collect();
bench.iter(move || {
for i in indexes.iter() {
let _ = blockstore.get_data_shred(slot, *i as u64);
let _ = blockstore.get_data_shred(slot, *i);
}
});
}
Expand Down
Loading