Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Bumped crc (#856)
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao authored Feb 20, 2022
1 parent 4d9e1e3 commit b6a6780
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 4 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ avro-schema = { version = "0.2", optional = true }
# compression of avro
libflate = { version = "1.1.1", optional = true }
snap = { version = "1", optional = true }
crc = { version = "1", optional = true }
crc = { version = "2", optional = true }
# async avro
async-stream = { version = "0.3.2", optional = true }

Expand Down
5 changes: 4 additions & 1 deletion src/io/avro/read/decompress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ use super::super::{Block, CompressedBlock};
use super::BlockStreamIterator;
use super::Compression;

const CRC_TABLE: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);

/// Decompresses an Avro block.
/// Returns whether the buffers where swapped.
pub fn decompress_block(
Expand Down Expand Up @@ -46,7 +48,8 @@ pub fn decompress_block(
.map_err(|e| ArrowError::ExternalFormat(e.to_string()))?;

let expected_crc = u32::from_be_bytes([crc[0], crc[1], crc[2], crc[3]]);
let actual_crc = crc::crc32::checksum_ieee(decompressed);

let actual_crc = CRC_TABLE.checksum(decompressed);
if expected_crc != actual_crc {
return Err(ArrowError::ExternalFormat(
"The crc of snap-compressed block does not match".to_string(),
Expand Down
5 changes: 3 additions & 2 deletions src/io/avro/write/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ use crate::error::Result;
use super::Compression;
use super::{Block, CompressedBlock};

const CRC_TABLE: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);

/// Compresses a [`Block`] to a [`CompressedBlock`].
pub fn compress(
block: &mut Block,
Expand Down Expand Up @@ -42,8 +44,7 @@ pub fn compress(
.map_err(|e| crate::error::ArrowError::ExternalFormat(e.to_string()))?;
compressed.truncate(compressed_bytes);

let crc = crc::crc32::checksum_ieee(block);
compressed.extend(crc.to_be_bytes());
compressed.extend(CRC_TABLE.checksum(block).to_be_bytes());
Ok(false)
}
#[cfg(not(feature = "io_avro_compression"))]
Expand Down

0 comments on commit b6a6780

Please sign in to comment.