From 9a47e2f005d4376f2a279b8edc22d494f4e4dd50 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 19 Nov 2024 16:44:55 +1000 Subject: [PATCH 1/2] Add PieceIndex::source_position() --- crates/subspace-core-primitives/src/pieces.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/subspace-core-primitives/src/pieces.rs b/crates/subspace-core-primitives/src/pieces.rs index 15a1564877..5f2f6eb91d 100644 --- a/crates/subspace-core-primitives/src/pieces.rs +++ b/crates/subspace-core-primitives/src/pieces.rs @@ -125,6 +125,14 @@ impl PieceIndex { (self.0 % ArchivedHistorySegment::NUM_PIECES as u64) as u32 } + /// Position of a source piece in the source pieces for a segment. + /// Panics if the piece is not a source piece. + #[inline] + pub const fn source_position(&self) -> u32 { + assert!(self.is_source()); + self.position() / (Self::source_ratio() as u32) + } + /// Is this piece index a source piece? #[inline] pub const fn is_source(&self) -> bool { From 428e22121ed80b5b235cd90d513dc5be3ef730d2 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 19 Nov 2024 15:54:24 +1000 Subject: [PATCH 2/2] Fix offset calculation bugs in ObjectFetcher --- shared/subspace-data-retrieval/src/object_fetcher.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/shared/subspace-data-retrieval/src/object_fetcher.rs b/shared/subspace-data-retrieval/src/object_fetcher.rs index 4968c86739..59436181ee 100644 --- a/shared/subspace-data-retrieval/src/object_fetcher.rs +++ b/shared/subspace-data-retrieval/src/object_fetcher.rs @@ -239,7 +239,7 @@ impl ObjectFetcher { // // The last 2 bytes might contain padding if a piece is the last piece in the segment. let before_last_two_bytes = piece_offset as usize <= RawRecord::SIZE - 1 - 2; - let piece_position_in_segment = piece_index.position(); + let piece_position_in_segment = piece_index.source_position(); let data_shards = RecordedHistorySegment::NUM_RAW_RECORDS as u32; let last_data_piece_in_segment = piece_position_in_segment >= data_shards - 1; @@ -261,7 +261,11 @@ impl ObjectFetcher { // // The last 2 bytes might contain padding if a piece is the last piece in the segment. let bytes_available_in_segment = - (data_shards - piece_position_in_segment) * RawRecord::SIZE as u32 - piece_offset - 2; + (data_shards - piece_position_in_segment) * RawRecord::SIZE as u32 - piece_offset; + let Some(bytes_available_in_segment) = bytes_available_in_segment.checked_sub(2) else { + // We need to reconstruct the full segment and discard padding before reading the length. + return Ok(None); + }; // Data from pieces that were already read, starting with piece at index `piece_index` let mut read_records_data = Vec::::with_capacity(RawRecord::SIZE * 2); @@ -374,7 +378,7 @@ impl ObjectFetcher { piece_offset: u32, ) -> Result, Error> { let mut segment_index = piece_index.segment_index(); - let piece_position_in_segment = piece_index.position(); + let piece_position_in_segment = piece_index.source_position(); // Used to access the data after it is converted to raw bytes let offset_in_segment = piece_position_in_segment as usize * RawRecord::SIZE + piece_offset as usize;