Directory restructure (#1532)

## Issue Addressed Closes #1487 Closes #1427 ## Proposed Changes Directory restructure in accordance with #1487. Also has temporary migration code to move the old directories into new structure. Also extracts all default directory names and utility functions into a `directory` crate to avoid repetitio. ## Additional info ~Since `validator_definition.yaml` stores absolute paths, users will have to manually change the keystore paths or delete the file to get the validators picked up by the vc.~. `validator_definition.yaml` is migrated as well from the default directories. Co-authored-by: realbigsean <[email protected]> Co-authored-by: Paul Hauner <[email protected]>
sigp · Sep 29, 2020 · 2ad3a09 · 2ad3a09
1 parent 0924a60
commit 2ad3a09
Show file tree

Hide file tree

Showing 40 changed files with 3,171 additions and 3,077 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -20,6 +20,7 @@ members = [
     "common/compare_fields",
     "common/compare_fields_derive",
     "common/deposit_contract",
+    "common/directory",
     "common/eth2_config",
     "common/eth2_interop_keypairs",
     "common/eth2_testnet_config",

diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml
@@ -24,6 +24,7 @@ eth2_testnet_config = { path = "../common/eth2_testnet_config" }
 web3 = "0.11.0"
 futures = { version = "0.3.5", features = ["compat"] }
 clap_utils = { path = "../common/clap_utils" }
+directory = { path = "../common/directory" }
 eth2_wallet = { path = "../crypto/eth2_wallet" }
 eth2_wallet_manager = { path = "../common/eth2_wallet_manager" }
 rand = "0.7.2"

diff --git a/account_manager/src/common.rs b/account_manager/src/common.rs
@@ -1,37 +1,15 @@
 use account_utils::PlainText;
 use account_utils::{read_input_from_user, strip_off_newlines};
-use clap::ArgMatches;
 use eth2_wallet::bip39::{Language, Mnemonic};
 use std::fs;
-use std::fs::create_dir_all;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::str::from_utf8;
 use std::thread::sleep;
 use std::time::Duration;
 
 pub const MNEMONIC_PROMPT: &str = "Enter the mnemonic phrase:";
 pub const WALLET_NAME_PROMPT: &str = "Enter wallet name:";
 
-pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
-    let path = path.as_ref();
-
-    if !path.exists() {
-        create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?;
-    }
-
-    Ok(())
-}
-
-pub fn base_wallet_dir(matches: &ArgMatches, arg: &'static str) -> Result<PathBuf, String> {
-    clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        arg,
-        PathBuf::new().join(".lighthouse").join("wallets"),
-    )
-}
-
-/// Reads in a mnemonic from the user. If the file path is provided, read from it. Otherwise, read
-/// from an interactive prompt using tty, unless the `--stdin-inputs` flag is provided.
 pub fn read_mnemonic_from_cli(
     mnemonic_path: Option<PathBuf>,
     stdin_inputs: bool,

diff --git a/account_manager/src/lib.rs b/account_manager/src/lib.rs
@@ -10,7 +10,7 @@ use types::EthSpec;
 pub const CMD: &str = "account_manager";
 pub const SECRETS_DIR_FLAG: &str = "secrets-dir";
 pub const VALIDATOR_DIR_FLAG: &str = "validator-dir";
-pub const BASE_DIR_FLAG: &str = "base-dir";
+pub const WALLETS_DIR_FLAG: &str = "wallets-dir";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)

diff --git a/account_manager/src/validator/create.rs b/account_manager/src/validator/create.rs
@@ -1,10 +1,13 @@
 use crate::common::read_wallet_name_from_cli;
 use crate::wallet::create::STDIN_INPUTS_FLAG;
-use crate::{common::ensure_dir_exists, SECRETS_DIR_FLAG, VALIDATOR_DIR_FLAG};
+use crate::{SECRETS_DIR_FLAG, WALLETS_DIR_FLAG};
 use account_utils::{
     random_password, read_password_from_user, strip_off_newlines, validator_definitions, PlainText,
 };
 use clap::{App, Arg, ArgMatches};
+use directory::{
+    ensure_dir_exists, parse_path_or_default_with_flag, DEFAULT_SECRET_DIR, DEFAULT_WALLET_DIR,
+};
 use environment::Environment;
 use eth2_wallet_manager::WalletManager;
 use std::ffi::OsStr;
@@ -14,7 +17,6 @@ use types::EthSpec;
 use validator_dir::Builder as ValidatorDirBuilder;
 
 pub const CMD: &str = "create";
-pub const BASE_DIR_FLAG: &str = "base-dir";
 pub const WALLET_NAME_FLAG: &str = "wallet-name";
 pub const WALLET_PASSWORD_FLAG: &str = "wallet-password";
 pub const DEPOSIT_GWEI_FLAG: &str = "deposit-gwei";
@@ -44,23 +46,22 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
+            Arg::with_name(WALLETS_DIR_FLAG)
+                .long(WALLETS_DIR_FLAG)
+                .value_name(WALLETS_DIR_FLAG)
+                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/{testnet}/wallets")
+                .takes_value(true)
+                .conflicts_with("datadir"),
         )
         .arg(
             Arg::with_name(SECRETS_DIR_FLAG)
                 .long(SECRETS_DIR_FLAG)
                 .value_name("SECRETS_DIR")
                 .help(
                     "The path where the validator keystore passwords will be stored. \
-                    Defaults to ~/.lighthouse/secrets",
+                    Defaults to ~/.lighthouse/{testnet}/secrets",
                 )
+                .conflicts_with("datadir")
                 .takes_value(true),
         )
         .arg(
@@ -111,23 +112,25 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 pub fn cli_run<T: EthSpec>(
     matches: &ArgMatches,
     mut env: Environment<T>,
-    wallet_base_dir: PathBuf,
+    validator_dir: PathBuf,
 ) -> Result<(), String> {
     let spec = env.core_context().eth2_config.spec;
 
     let name: Option<String> = clap_utils::parse_optional(matches, WALLET_NAME_FLAG)?;
     let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
+    let wallet_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_WALLET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, WALLETS_DIR_FLAG, DEFAULT_WALLET_DIR)?
+    };
+    let secrets_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_SECRET_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, SECRETS_DIR_FLAG, DEFAULT_SECRET_DIR)?
+    };
 
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-    let secrets_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        SECRETS_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("secrets"),
-    )?;
     let deposit_gwei = clap_utils::parse_optional(matches, DEPOSIT_GWEI_FLAG)?
         .unwrap_or_else(|| spec.max_effective_balance);
     let count: Option<usize> = clap_utils::parse_optional(matches, COUNT_FLAG)?;
@@ -136,6 +139,9 @@ pub fn cli_run<T: EthSpec>(
     ensure_dir_exists(&validator_dir)?;
     ensure_dir_exists(&secrets_dir)?;
 
+    eprintln!("secrets-dir path {:?}", secrets_dir);
+    eprintln!("wallets-dir path {:?}", wallet_base_dir);
+
     let starting_validator_count = existing_validator_count(&validator_dir)?;
 
     let n = match (count, at_most) {
@@ -166,7 +172,7 @@ pub fn cli_run<T: EthSpec>(
     let wallet_password = read_wallet_password_from_cli(wallet_password_path, stdin_inputs)?;
 
     let mgr = WalletManager::open(&wallet_base_dir)
-        .map_err(|e| format!("Unable to open --{}: {:?}", BASE_DIR_FLAG, e))?;
+        .map_err(|e| format!("Unable to open --{}: {:?}", WALLETS_DIR_FLAG, e))?;
 
     let mut wallet = mgr
         .wallet_by_name(&wallet_name)

diff --git a/account_manager/src/validator/deposit.rs b/account_manager/src/validator/deposit.rs
@@ -46,16 +46,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
             The deposit contract address will be determined by the --testnet-dir flag on the \
             primary Lighthouse binary.",
         )
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to the validator client data directory. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
         .arg(
             Arg::with_name(VALIDATOR_FLAG)
                 .long(VALIDATOR_FLAG)
@@ -209,14 +199,10 @@ where
 pub fn cli_run<T: EthSpec>(
     matches: &ArgMatches<'_>,
     mut env: Environment<T>,
+    validator_dir: PathBuf,
 ) -> Result<(), String> {
     let log = env.core_context().log().clone();
 
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
     let validator: String = clap_utils::parse_required(matches, VALIDATOR_FLAG)?;
     let eth1_ipc_path: Option<PathBuf> = clap_utils::parse_optional(matches, ETH1_IPC_FLAG)?;
     let eth1_http_url: Option<String> = clap_utils::parse_optional(matches, ETH1_HTTP_FLAG)?;
@@ -225,7 +211,7 @@ pub fn cli_run<T: EthSpec>(
     let confirmation_batch_size: usize =
         clap_utils::parse_required(matches, CONFIRMATION_BATCH_SIZE_FLAG)?;
 
-    let manager = ValidatorManager::open(&data_dir)
+    let manager = ValidatorManager::open(&validator_dir)
         .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
 
     let validators = match validator.as_ref() {

diff --git a/account_manager/src/validator/import.rs b/account_manager/src/validator/import.rs
@@ -1,5 +1,4 @@
 use crate::wallet::create::STDIN_INPUTS_FLAG;
-use crate::{common::ensure_dir_exists, VALIDATOR_DIR_FLAG};
 use account_utils::{
     eth2_keystore::Keystore,
     read_password_from_user,
@@ -55,16 +54,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .required_unless(KEYSTORE_FLAG)
                 .takes_value(true),
         )
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path where the validator directories will be created. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
         .arg(
             Arg::with_name(STDIN_INPUTS_FLAG)
                 .long(STDIN_INPUTS_FLAG)
@@ -77,19 +66,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         )
 }
 
-pub fn cli_run(matches: &ArgMatches) -> Result<(), String> {
+pub fn cli_run(matches: &ArgMatches, validator_dir: PathBuf) -> Result<(), String> {
     let keystore: Option<PathBuf> = clap_utils::parse_optional(matches, KEYSTORE_FLAG)?;
     let keystores_dir: Option<PathBuf> = clap_utils::parse_optional(matches, DIR_FLAG)?;
-    let validator_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
     let stdin_inputs = matches.is_present(STDIN_INPUTS_FLAG);
     let reuse_password = matches.is_present(REUSE_PASSWORD_FLAG);
 
-    ensure_dir_exists(&validator_dir)?;
-
     let mut defs = ValidatorDefinitions::open_or_create(&validator_dir)
         .map_err(|e| format!("Unable to open {}: {:?}", CONFIG_FILENAME, e))?;
 

diff --git a/account_manager/src/validator/list.rs b/account_manager/src/validator/list.rs
@@ -1,38 +1,21 @@
 use crate::VALIDATOR_DIR_FLAG;
-use clap::{App, Arg, ArgMatches};
+use clap::App;
 use std::path::PathBuf;
 use validator_dir::Manager as ValidatorManager;
 
 pub const CMD: &str = "list";
 
 pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
-    App::new(CMD)
-        .arg(
-            Arg::with_name(VALIDATOR_DIR_FLAG)
-                .long(VALIDATOR_DIR_FLAG)
-                .value_name("VALIDATOR_DIRECTORY")
-                .help(
-                    "The path to search for validator directories. \
-                    Defaults to ~/.lighthouse/validators",
-                )
-                .takes_value(true),
-        )
-        .about("Lists the names of all validators.")
+    App::new(CMD).about("Lists the names of all validators.")
 }
 
-pub fn cli_run(matches: &ArgMatches<'_>) -> Result<(), String> {
-    let data_dir = clap_utils::parse_path_with_default_in_home_dir(
-        matches,
-        VALIDATOR_DIR_FLAG,
-        PathBuf::new().join(".lighthouse").join("validators"),
-    )?;
-
-    let mgr = ValidatorManager::open(&data_dir)
+pub fn cli_run(validator_dir: PathBuf) -> Result<(), String> {
+    let mgr = ValidatorManager::open(&validator_dir)
         .map_err(|e| format!("Unable to read --{}: {:?}", VALIDATOR_DIR_FLAG, e))?;
 
     for (name, _path) in mgr
         .directory_names()
-        .map_err(|e| format!("Unable to list wallets: {:?}", e))?
+        .map_err(|e| format!("Unable to list validators: {:?}", e))?
     {
         println!("{}", name)
     }

diff --git a/account_manager/src/validator/mod.rs b/account_manager/src/validator/mod.rs
@@ -4,9 +4,11 @@ pub mod import;
 pub mod list;
 pub mod recover;
 
-use crate::common::base_wallet_dir;
+use crate::VALIDATOR_DIR_FLAG;
 use clap::{App, Arg, ArgMatches};
+use directory::{parse_path_or_default_with_flag, DEFAULT_VALIDATOR_DIR};
 use environment::Environment;
+use std::path::PathBuf;
 use types::EthSpec;
 
 pub const CMD: &str = "validator";
@@ -15,11 +17,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
     App::new(CMD)
         .about("Provides commands for managing Eth2 validators.")
         .arg(
-            Arg::with_name("base-dir")
-                .long("base-dir")
-                .value_name("BASE_DIRECTORY")
-                .help("A path containing Eth2 EIP-2386 wallets. Defaults to ~/.lighthouse/wallets")
-                .takes_value(true),
+            Arg::with_name(VALIDATOR_DIR_FLAG)
+                .long(VALIDATOR_DIR_FLAG)
+                .value_name("VALIDATOR_DIRECTORY")
+                .help(
+                    "The path to search for validator directories. \
+                    Defaults to ~/.lighthouse/{testnet}/validators",
+                )
+                .takes_value(true)
+                .global(true)
+                .conflicts_with("datadir"),
         )
         .subcommand(create::cli_app())
         .subcommand(deposit::cli_app())
@@ -29,14 +36,20 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn cli_run<T: EthSpec>(matches: &ArgMatches, env: Environment<T>) -> Result<(), String> {
-    let base_wallet_dir = base_wallet_dir(matches, "base-dir")?;
+    let validator_base_dir = if matches.value_of("datadir").is_some() {
+        let path: PathBuf = clap_utils::parse_required(matches, "datadir")?;
+        path.join(DEFAULT_VALIDATOR_DIR)
+    } else {
+        parse_path_or_default_with_flag(matches, VALIDATOR_DIR_FLAG, DEFAULT_VALIDATOR_DIR)?
+    };
+    eprintln!("validator-dir path: {:?}", validator_base_dir);
 
     match matches.subcommand() {
-        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, base_wallet_dir),
-        (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env),
-        (import::CMD, Some(matches)) => import::cli_run(matches),
-        (list::CMD, Some(matches)) => list::cli_run(matches),
-        (recover::CMD, Some(matches)) => recover::cli_run(matches),
+        (create::CMD, Some(matches)) => create::cli_run::<T>(matches, env, validator_base_dir),
+        (deposit::CMD, Some(matches)) => deposit::cli_run::<T>(matches, env, validator_base_dir),
+        (import::CMD, Some(matches)) => import::cli_run(matches, validator_base_dir),
+        (list::CMD, Some(_)) => list::cli_run(validator_base_dir),
+        (recover::CMD, Some(matches)) => recover::cli_run(matches, validator_base_dir),
         (unknown, _) => Err(format!(
             "{} does not have a {} command. See --help",
             CMD, unknown