diff --git a/CHANGELOG.md b/CHANGELOG.md index 48f7cf76c5..33b333b9f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Pearl changelog - Index regeneration checks record data checksum (#215) #### Changed +- `ring` crate replaced with `sha2` crate (#246) +- `async-std` replaced with `async-lock` to avoid redundant dependencies (#247) - Change locks for filters update (#218) - Serialization moved out of critical section (#188) - Removed multiple header checksum calculations (#206) diff --git a/Cargo.toml b/Cargo.toml index 7408d35892..e29061620a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,14 +27,10 @@ ahash = "=0.7.6" anyhow = "1.0" async-trait = "0.1" bincode = "1.3" -chrono = "0.4" -clap = { version = "3.2", optional = true } crc = "=3.0.0" -env_logger = "0.9" futures = "0.3" log = "0.4" -rand = "0.8" -ring = "0.16" +sha2 = "0.10" rio = "0.9.4" serde = "1.0" serde_derive = "1.0" @@ -44,7 +40,10 @@ nix = "0.25.0" libc = "0.2" static_assertions = "1.1.0" bytes = "1.2" -async-std = "1.12" +async-lock = "2.6" +# Benchmark only dependencies +clap = { version = "3.2", optional = true } +env_logger = { version = "0.9", optional = true } [dependencies.tokio] version = "1.21" @@ -52,7 +51,7 @@ features = ["fs", "io-util", "sync", "time", "rt", "macros", "rt-multi-thread"] [features] # default = ["benchmark"] -benchmark = ["clap"] +benchmark = ["clap", "env_logger"] [lib] name = "pearl" @@ -64,3 +63,8 @@ required-features = ["benchmark"] [build-dependencies] chrono = "0.4" + +[dev-dependencies] +env_logger = "0.9" +chrono = "0.4" +rand = "0.8" \ No newline at end of file diff --git a/src/blob/core.rs b/src/blob/core.rs index 47c3b3ba3e..98f6c7e053 100644 --- a/src/blob/core.rs +++ b/src/blob/core.rs @@ -1,6 +1,6 @@ use std::time::SystemTime; -use async_std::sync::RwLockUpgradableReadGuard; +use async_lock::RwLockUpgradableReadGuard; use bytes::{BufMut, Bytes, BytesMut}; use tokio::time::Instant; diff --git a/src/blob/index/bptree/core.rs b/src/blob/index/bptree/core.rs index bb0cbf7d1e..6a451f9e8d 100644 --- a/src/blob/index/bptree/core.rs +++ b/src/blob/index/bptree/core.rs @@ -397,13 +397,13 @@ where } fn hash_valid(header: &IndexHeader, buf: &mut [u8]) -> Result { - let hash = header.hash.clone(); + let hash = &header.hash; let mut header = header.clone(); - header.hash = vec![0; ring::digest::SHA256.output_len]; + header.reset_hash(); header.set_written(false); serialize_into(&mut buf[..], &header)?; - let new_hash = get_hash(&buf); - Ok(hash == new_hash) + let new_hash = IndexHashCalculator::get_hash(&buf); + Ok(*hash == new_hash) } async fn read_index_header(file: &File) -> Result { diff --git a/src/blob/index/bptree/serializer.rs b/src/blob/index/bptree/serializer.rs index 5e55c31e19..a478f6ecd8 100644 --- a/src/blob/index/bptree/serializer.rs +++ b/src/blob/index/bptree/serializer.rs @@ -230,7 +230,7 @@ where buf.extend_from_slice(&self.meta_buf); buf.extend_from_slice(&self.tree_buf); Self::append_headers(self.headers_btree, &mut buf)?; - let hash = get_hash(&buf); + let hash = IndexHashCalculator::get_hash(&buf); let header = IndexHeader::with_hash( self.header.record_header_size, self.header.records_count, diff --git a/src/blob/index/header.rs b/src/blob/index/header.rs index a84cd1fc60..cd0b997783 100644 --- a/src/blob/index/header.rs +++ b/src/blob/index/header.rs @@ -108,6 +108,14 @@ impl IndexHeader { pub(crate) fn blob_size(&self) -> u64 { self.blob_size } + + pub(crate) fn reset_hash(&mut self) { + if self.hash.len() == IndexHashCalculator::HASH_LENGTH { + self.hash.fill(0); + } else { + self.hash = vec![0; IndexHashCalculator::HASH_LENGTH]; + } + } } impl Default for IndexHeader { @@ -117,7 +125,7 @@ impl Default for IndexHeader { record_header_size: 0, meta_size: 0, blob_size: 0, - hash: vec![0; ring::digest::SHA256.output_len], + hash: vec![0; IndexHashCalculator::HASH_LENGTH], version: HEADER_VERSION << 1, key_size: 0, magic_byte: INDEX_HEADER_MAGIC_BYTE, diff --git a/src/blob/index/simple.rs b/src/blob/index/simple.rs index 1f9fff7a51..9a70f1293e 100644 --- a/src/blob/index/simple.rs +++ b/src/blob/index/simple.rs @@ -160,13 +160,13 @@ where // helpers impl SimpleFileIndex { fn hash_valid(header: &IndexHeader, buf: &mut [u8]) -> Result { - let hash = header.hash.clone(); + let hash = &header.hash; let mut header = header.clone(); - header.hash = vec![0; ring::digest::SHA256.output_len]; + header.reset_hash(); header.set_written(false); serialize_into(&mut buf[..], &header)?; - let new_hash = get_hash(&buf); - Ok(hash == new_hash) + let new_hash = IndexHashCalculator::get_hash(&buf); + Ok(*hash == new_hash) } async fn read_index_header(file: &File) -> Result { @@ -348,7 +348,7 @@ impl SimpleFileIndex { "blob index simple serialize headers buf len after: {}", buf.len() ); - let hash = get_hash(&buf); + let hash = IndexHashCalculator::get_hash(&buf); let header = IndexHeader::with_hash( record_header_size, headers.len(), diff --git a/src/blob/index/tools.rs b/src/blob/index/tools.rs index 25822bae42..2c7df981e3 100644 --- a/src/blob/index/tools.rs +++ b/src/blob/index/tools.rs @@ -1,12 +1,17 @@ use super::prelude::*; use std::mem::size_of; -pub(crate) fn get_hash(buf: &[u8]) -> Vec { - use ring::digest::{Context, SHA256}; - let mut context = Context::new(&SHA256); - context.update(buf); - let digest = context.finish(); - digest.as_ref().to_vec() +/// Hash calculation helper for index +pub(crate) struct IndexHashCalculator; + +impl IndexHashCalculator { + pub(crate) const HASH_LENGTH: usize = 32; + + pub(crate) fn get_hash(buf: &[u8]) -> Vec { + use sha2::{Sha256, Digest}; + let digest = Sha256::digest(buf); + digest.to_vec() + } } // if there is no elements, data will be wrong (because we can't get key_size), @@ -46,3 +51,19 @@ pub(crate) fn clean_file(path: impl AsRef, recreate_index_file: bool) -> R Err(anyhow!(msg)) } } + + +#[cfg(test)] +mod tests { + use super::IndexHashCalculator; + + #[test] + pub fn test_hash_compatibility() { + let data_vec: Vec = (0..1024).into_iter().map(|i| (i % 256) as u8).collect(); + // SHA256 hash calculated with ring crate + let expected_hash = vec![120, 91, 7, 81, 252, 44, 83, 220, 20, 164, 206, 61, 128, 14, 105, 239, 156, 225, 0, 158, 179, 39, 204, 244, 88, 175, 224, 156, 36, 44, 38, 201]; + let actual_hash = IndexHashCalculator::get_hash(&data_vec); + + assert_eq!(expected_hash, actual_hash); + } +} \ No newline at end of file diff --git a/src/blob/mod.rs b/src/blob/mod.rs index 87d67b8c1a..84e0baa970 100644 --- a/src/blob/mod.rs +++ b/src/blob/mod.rs @@ -15,7 +15,7 @@ pub(crate) use super::prelude::*; mod prelude { pub(crate) use super::*; - pub(crate) use async_std::sync::{ + pub(crate) use async_lock::{ RwLock as ASRwLock, RwLockUpgradableReadGuard as ASRwLockUpgradableReadGuard, }; pub(crate) use index::Index; diff --git a/src/lib.rs b/src/lib.rs index b1a57fbd67..1fd5f89d5a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,7 +45,6 @@ extern crate serde_derive; extern crate anyhow; extern crate bytes; -extern crate ring; /// Basic info about current build. pub mod build_info; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 16c4457a95..e621303d61 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -16,7 +16,7 @@ pub use self::{ }; mod prelude { - pub(crate) use async_std::sync::RwLock as ASRwLock; + pub(crate) use async_lock::RwLock as ASRwLock; pub(crate) use { super::{ config::Config, core::Inner, observer::Msg, observer::Observer,