Skip to content

Commit

Permalink
Support opening an in-use rocksdb as secondary (#10209)
Browse files Browse the repository at this point in the history 
automerge

(cherry picked from commit caa7f7a)

# Conflicts:
#	ledger-tool/Cargo.toml
#	ledger-tool/src/main.rs
  • Loading branch information
@ryoqun
ryoqun authored and mergify-bot committed Jun 3, 2020
1 parent 5ade9b9 commit e319e0a
Show file tree
Hide file tree
Showing 7 changed files with 237 additions and 54 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions genesis/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use solana_clap_utils::{
};
use solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account};
use solana_ledger::{
blockstore::create_new_ledger, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
poh::compute_hashes_per_tick,
blockstore::create_new_ledger, blockstore_db::AccessType,
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, poh::compute_hashes_per_tick,
};
use solana_sdk::{
account::Account,
Expand Down Expand Up @@ -543,6 +543,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
&ledger_path,
&genesis_config,
max_genesis_archive_unpacked_size,
AccessType::PrimaryOnly,
)?;

println!("{}", genesis_config);
Expand Down
16 changes: 16 additions & 0 deletions ledger-tool/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ homepage = "https://solana.com/"
bs58 = "0.3.0"
clap = "2.33.0"
histogram = "*"
<<<<<<< HEAD
serde_json = "1.0.48"
serde_yaml = "0.8.11"
solana-clap-utils = { path = "../clap-utils", version = "1.1.16" }
Expand All @@ -23,6 +24,21 @@ solana-sdk = { path = "../sdk", version = "1.1.16" }
solana-stake-program = { path = "../programs/stake", version = "1.1.16" }
solana-transaction-status = { path = "../transaction-status", version = "1.1.16" }
solana-vote-program = { path = "../programs/vote", version = "1.1.16" }
=======
log = { version = "0.4.8" }
serde_json = "1.0.53"
serde_yaml = "0.8.12"
solana-clap-utils = { path = "../clap-utils", version = "1.3.0" }
solana-cli = { path = "../cli", version = "1.3.0" }
solana-ledger = { path = "../ledger", version = "1.3.0" }
solana-logger = { path = "../logger", version = "1.3.0" }
solana-runtime = { path = "../runtime", version = "1.3.0" }
solana-sdk = { path = "../sdk", version = "1.3.0" }
solana-stake-program = { path = "../programs/stake", version = "1.3.0" }
solana-transaction-status = { path = "../transaction-status", version = "1.3.0" }
solana-version = { path = "../version", version = "1.3.0" }
solana-vote-program = { path = "../programs/vote", version = "1.3.0" }
>>>>>>> caa7f7a0c... Support opening an in-use rocksdb as secondary (#10209)
tempfile = "3.1.0"

[dev-dependencies]
Expand Down
122 changes: 102 additions & 20 deletions ledger-tool/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,13 @@ use solana_ledger::{
bank_forks::{BankForks, SnapshotConfig},
bank_forks_utils,
blockstore::Blockstore,
<<<<<<< HEAD
blockstore_db::{self, Column, Database},
blockstore_processor::{BankForksInfo, ProcessOptions},
=======
blockstore_db::{self, AccessType, Column, Database},
blockstore_processor::ProcessOptions,
>>>>>>> caa7f7a0c... Support opening an in-use rocksdb as secondary (#10209)
hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
rooted_slot_iterator::RootedSlotIterator,
snapshot_utils,
Expand All @@ -30,6 +35,8 @@ use std::{
str::FromStr,
};

use log::*;

#[derive(PartialEq)]
enum LedgerOutputMethod {
Print,
Expand Down Expand Up @@ -494,8 +501,8 @@ fn analyze_storage(database: &Database) -> Result<(), String> {
Ok(())
}

fn open_blockstore(ledger_path: &Path) -> Blockstore {
match Blockstore::open(ledger_path) {
fn open_blockstore(ledger_path: &Path, access_type: AccessType) -> Blockstore {
match Blockstore::open_with_access_type(ledger_path, access_type) {
Ok(blockstore) => blockstore,
Err(err) => {
eprintln!("Failed to open ledger at {:?}: {:?}", ledger_path, err);
Expand All @@ -504,8 +511,8 @@ fn open_blockstore(ledger_path: &Path) -> Blockstore {
}
}

fn open_database(ledger_path: &Path) -> Database {
match Database::open(&ledger_path.join("rocksdb")) {
fn open_database(ledger_path: &Path, access_type: AccessType) -> Database {
match Database::open(&ledger_path.join("rocksdb"), access_type) {
Ok(database) => database,
Err(err) => {
eprintln!("Unable to read the Ledger rocksdb: {:?}", err);
Expand All @@ -528,6 +535,7 @@ fn load_bank_forks(
ledger_path: &PathBuf,
genesis_config: &GenesisConfig,
process_options: ProcessOptions,
access_type: AccessType,
) -> bank_forks_utils::LoadResult {
let snapshot_config = if arg_matches.is_present("no_snapshot") {
None
Expand All @@ -538,15 +546,29 @@ fn load_bank_forks(
snapshot_path: ledger_path.clone().join("snapshot"),
})
};
let blockstore = open_blockstore(&ledger_path, access_type);
let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
if !blockstore.is_primary_access() {
// Be defenstive, when default account dir is explicitly specified, it's still possible
// to wipe the dir possibly shared by the running validator!
eprintln!("Error: custom accounts path is not supported under secondary access");
exit(1);
}
account_paths.split(',').map(PathBuf::from).collect()
} else {
} else if blockstore.is_primary_access() {
vec![ledger_path.join("accounts")]
} else {
let non_primary_accounts_path = ledger_path.join("accounts.ledger-tool");
warn!(
"Default accounts path is switched aligning with Blockstore's secondary access: {:?}",
non_primary_accounts_path
);
vec![non_primary_accounts_path]
};

bank_forks_utils::load(
&genesis_config,
&open_blockstore(&ledger_path),
&blockstore,
account_paths,
snapshot_config.as_ref(),
process_options,
Expand Down Expand Up @@ -819,7 +841,7 @@ fn main() {
("print", Some(arg_matches)) => {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
output_ledger(
open_blockstore(&ledger_path),
open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary),
starting_slot,
LedgerOutputMethod::Print,
);
Expand All @@ -841,11 +863,22 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
<<<<<<< HEAD
match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
let bank_info = &bank_forks_info[0];
let bank = bank_forks[bank_info.bank_slot].clone();

=======
match load_bank_forks(
arg_matches,
&ledger_path,
&genesis_config,
process_options,
AccessType::TryPrimaryThenSecondary,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
>>>>>>> caa7f7a0c... Support opening an in-use rocksdb as secondary (#10209)
println!(
"{}",
compute_shred_version(
Expand All @@ -862,7 +895,12 @@ fn main() {
}
("slot", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
<<<<<<< HEAD
let blockstore = open_blockstore(&ledger_path);
=======
let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
let blockstore = open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary);
>>>>>>> caa7f7a0c... Support opening an in-use rocksdb as secondary (#10209)
for slot in slots {
println!("Slot {}", slot);
if let Err(err) = output_slot(&blockstore, slot, &LedgerOutputMethod::Print) {
Expand All @@ -873,14 +911,14 @@ fn main() {
("json", Some(arg_matches)) => {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
output_ledger(
open_blockstore(&ledger_path),
open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary),
starting_slot,
LedgerOutputMethod::Json,
);
}
("set-dead-slot", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let blockstore = open_blockstore(&ledger_path);
let blockstore = open_blockstore(&ledger_path, AccessType::PrimaryOnly);
for slot in slots {
match blockstore.set_dead_slot(slot) {
Ok(_) => println!("Slot {} dead", slot),
Expand All @@ -905,6 +943,7 @@ fn main() {
&ledger_path,
&open_genesis_config_by(&ledger_path, arg_matches),
process_options,
AccessType::TryPrimaryThenSecondary,
)
.unwrap_or_else(|err| {
eprintln!("Ledger verification failed: {:?}", err);
Expand All @@ -927,6 +966,7 @@ fn main() {
&ledger_path,
&open_genesis_config_by(&ledger_path, arg_matches),
process_options,
AccessType::TryPrimaryThenSecondary,
) {
Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
let dot = graph_forks(
Expand Down Expand Up @@ -967,8 +1007,19 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
<<<<<<< HEAD
match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
Ok((bank_forks, _bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
=======
match load_bank_forks(
arg_matches,
&ledger_path,
&genesis_config,
process_options,
AccessType::TryPrimaryThenSecondary,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
>>>>>>> caa7f7a0c... Support opening an in-use rocksdb as secondary (#10209)
let bank = bank_forks.get(snapshot_slot).unwrap_or_else(|| {
eprintln!("Error: Slot {} is not available", snapshot_slot);
exit(1);
Expand Down Expand Up @@ -1031,6 +1082,7 @@ fn main() {
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let include_sysvars = arg_matches.is_present("include_sysvars");
<<<<<<< HEAD
match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
let slot = dev_halt_at_slot.unwrap_or_else(|| {
Expand All @@ -1041,6 +1093,17 @@ fn main() {
bank_forks_info[0].bank_slot
});

=======
match load_bank_forks(
arg_matches,
&ledger_path,
&genesis_config,
process_options,
AccessType::TryPrimaryThenSecondary,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let slot = bank_forks.working_bank().slot();
>>>>>>> caa7f7a0c... Support opening an in-use rocksdb as secondary (#10209)
let bank = bank_forks.get(slot).unwrap_or_else(|| {
eprintln!("Error: Slot {} is not available", slot);
exit(1);
Expand Down Expand Up @@ -1080,6 +1143,7 @@ fn main() {
..ProcessOptions::default()
};
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
<<<<<<< HEAD
match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
let slot = dev_halt_at_slot.unwrap_or_else(|| {
Expand All @@ -1090,6 +1154,17 @@ fn main() {
bank_forks_info[0].bank_slot
});

=======
match load_bank_forks(
arg_matches,
&ledger_path,
&genesis_config,
process_options,
AccessType::TryPrimaryThenSecondary,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let slot = bank_forks.working_bank().slot();
>>>>>>> caa7f7a0c... Support opening an in-use rocksdb as secondary (#10209)
let bank = bank_forks.get(slot).unwrap_or_else(|| {
eprintln!("Error: Slot {} is not available", slot);
exit(1);
Expand Down Expand Up @@ -1151,12 +1226,12 @@ fn main() {
("purge", Some(arg_matches)) => {
let start_slot = value_t_or_exit!(arg_matches, "start_slot", Slot);
let end_slot = value_t_or_exit!(arg_matches, "end_slot", Slot);
let blockstore = open_blockstore(&ledger_path);
let blockstore = open_blockstore(&ledger_path, AccessType::PrimaryOnly);
blockstore.purge_slots(start_slot, end_slot);
blockstore.purge_from_next_slots(start_slot, end_slot);
}
("list-roots", Some(arg_matches)) => {
let blockstore = open_blockstore(&ledger_path);
let blockstore = open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary);
let max_height = if let Some(height) = arg_matches.value_of("max_height") {
usize::from_str(height).expect("Maximum height must be a number")
} else {
Expand Down Expand Up @@ -1209,7 +1284,9 @@ fn main() {
});
}
("bounds", Some(arg_matches)) => {
match open_blockstore(&ledger_path).slot_meta_iterator(0) {
match open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary)
.slot_meta_iterator(0)
{
Ok(metas) => {
let all = arg_matches.is_present("all");

Expand All @@ -1235,15 +1312,20 @@ fn main() {
}
}
}
("analyze-storage", _) => match analyze_storage(&open_database(&ledger_path)) {
Ok(()) => {
println!("Ok.");
}
Err(err) => {
eprintln!("Unable to read the Ledger: {:?}", err);
exit(1);
("analyze-storage", _) => {
match analyze_storage(&open_database(
&ledger_path,
AccessType::TryPrimaryThenSecondary,
)) {
Ok(()) => {
println!("Ok.");
}
Err(err) => {
eprintln!("Unable to read the Ledger: {:?}", err);
exit(1);
}
}
},
}
("", _) => {
eprintln!("{}", matches.usage());
exit(1);
Expand Down
Loading

0 comments on commit e319e0a

Please sign in to comment.