Skip to content

Commit

Permalink
[Storage] Implement shard id extraction in a different way. (#13940)
Browse files Browse the repository at this point in the history
  • Loading branch information
grao1991 authored Jul 9, 2024
1 parent 5b20c16 commit 68f047c
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 21 deletions.
12 changes: 0 additions & 12 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -643,7 +643,6 @@ num_cpus = "1.13.1"
num-derive = "0.3.3"
num-integer = "0.1.42"
num-traits = "0.2.15"
number_range = "0.3.2"
once_cell = "1.10.0"
ordered-float = "3.9.1"
ouroboros = "0.15.6"
Expand Down
1 change: 0 additions & 1 deletion config/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ cfg-if = { workspace = true }
get_if_addrs = { workspace = true }
maplit = { workspace = true }
num_cpus = { workspace = true }
number_range = { workspace = true }
poem-openapi = { workspace = true }
rand = { workspace = true }
serde = { workspace = true }
Expand Down
35 changes: 28 additions & 7 deletions config/src/config/storage_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,16 @@ use crate::{
config::{config_sanitizer::ConfigSanitizer, node_config_loader::NodeType, Error, NodeConfig},
utils,
};
use anyhow::{ensure, Result};
use anyhow::{bail, ensure, Result};
use aptos_logger::warn;
use aptos_types::chain_id::ChainId;
use arr_macro::arr;
use number_range::NumberRangeOptions;
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
net::{IpAddr, Ipv4Addr, SocketAddr},
path::{Path, PathBuf},
str::FromStr,
};

// Lru cache will consume about 2G RAM based on this default value.
Expand Down Expand Up @@ -49,11 +49,7 @@ impl ShardedDbPathConfig {
pub fn get_shard_paths(&self) -> Result<HashMap<u8, PathBuf>> {
let mut result = HashMap::new();
for shard_path in &self.shard_paths {
let shard_ids = NumberRangeOptions::<u8>::new()
.with_list_sep(',')
.with_range_sep('-')
.parse(shard_path.shards.as_str())?
.collect::<Vec<u8>>();
let shard_ids = Self::parse(shard_path.shards.as_str())?;
let path = &shard_path.path;
ensure!(
path.is_absolute(),
Expand All @@ -74,6 +70,31 @@ impl ShardedDbPathConfig {

Ok(result)
}

fn parse(path: &str) -> Result<Vec<u8>> {
let mut shard_ids = vec![];
for p in path.split(',') {
let num_or_range: Vec<&str> = p.split('-').collect();
match num_or_range.len() {
1 => {
let num = u8::from_str(num_or_range[0])?;
ensure!(num < 16);
shard_ids.push(num);
},
2 => {
let range_start = u8::from_str(num_or_range[0])?;
let range_end = u8::from_str(num_or_range[1])?;
ensure!(range_start <= range_end && range_end < 16);
for num in range_start..=range_end {
shard_ids.push(num);
}
},
_ => bail!("Invalid path: {path}."),
}
}

Ok(shard_ids)
}
}

/// Port selected RocksDB options for tuning underlying rocksdb instance of AptosDB.
Expand Down

0 comments on commit 68f047c

Please sign in to comment.