Add lighthouse db command (#3129)

## Proposed Changes Add a `lighthouse db` command with three initial subcommands: - `lighthouse db version`: print the database schema version. - `lighthouse db migrate --to N`: manually upgrade (or downgrade!) the database to a different version. - `lighthouse db inspect --column C`: log the key and size in bytes of every value in a given `DBColumn`. This PR lays the groundwork for other changes, namely: - Mark's fast-deposit sync (#2915), for which I think we should implement a database downgrade (from v9 to v8). - My `tree-states` work, which already implements a downgrade (v10 to v8). - Standalone purge commands like `lighthouse db purge-dht` per #2824. ## Additional Info I updated the `strum` crate to 0.24.0, which necessitated some changes in the network code to remove calls to deprecated methods. Thanks to @winksaville for the motivation, and implementation work that I used as a source of inspiration (#2685).
sigp · Apr 1, 2022 · 41e7a07 · 41e7a07
1 parent ea78336
commit 41e7a07
Show file tree

Hide file tree

Showing 25 changed files with 446 additions and 86 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -14,7 +14,7 @@ members = [
     "beacon_node/store",
     "beacon_node/timer",
 
-	"boot_node",
+    "boot_node",
 
     "common/account_utils",
     "common/clap_utils",
@@ -45,6 +45,8 @@ members = [
     "common/fallback",
     "common/monitoring_api",
 
+    "database_manager",
+
     "consensus/cached_tree_hash",
     "consensus/int_to_bytes",
     "consensus/fork_choice",

diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml
@@ -55,7 +55,7 @@ derivative = "2.1.1"
 itertools = "0.10.0"
 slasher = { path = "../../slasher" }
 eth2 = { path = "../../common/eth2" }
-strum = { version = "0.21.0", features = ["derive"] }
+strum = { version = "0.24.0", features = ["derive"] }
 logging = { path = "../../common/logging" }
 execution_layer = { path = "../execution_layer" }
 sensitive_url = { path = "../../common/sensitive_url" }

diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs
@@ -169,7 +169,7 @@ impl Config {
     /// For more information, see:
     ///
     /// https://github.com/sigp/lighthouse/pull/2843
-    fn get_data_dir(&self) -> PathBuf {
+    pub fn get_data_dir(&self) -> PathBuf {
         let existing_legacy_dir = self.get_existing_legacy_data_dir();
 
         if let Some(legacy_dir) = existing_legacy_dir {

diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml
@@ -34,4 +34,4 @@ tempfile = "3.1.0"
 rand = "0.7.3"
 zeroize = { version = "1.4.2", features = ["zeroize_derive"] }
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
-lazy_static = "1.4.0"
+lazy_static = "1.4.0"
diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml
@@ -36,7 +36,7 @@ task_executor = { path = "../../common/task_executor" }
 rand = "0.7.3"
 directory = { path = "../../common/directory" }
 regex = "1.5.5"
-strum = { version = "0.21.0", features = ["derive"] }
+strum = { version = "0.24.0", features = ["derive"] }
 superstruct = "0.4.1"
 prometheus-client = "0.15.0"
 unused_port = { path = "../../common/unused_port" }

diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs
@@ -388,7 +388,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
     /// Updates `PeerInfo` with `identify` information.
     pub fn identify(&mut self, peer_id: &PeerId, info: &IdentifyInfo) {
         if let Some(peer_info) = self.network_globals.peers.write().peer_info_mut(peer_id) {
-            let previous_kind = peer_info.client().kind.clone();
+            let previous_kind = peer_info.client().kind;
             let previous_listening_addresses =
                 peer_info.set_listening_addresses(info.listen_addrs.clone());
             peer_info.set_client(peerdb::client::Client::from_identify_info(info));
@@ -412,12 +412,9 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
                 ) {
                     metrics::inc_gauge_vec(
                         &metrics::PEERS_PER_CLIENT,
-                        &[&peer_info.client().kind.to_string()],
-                    );
-                    metrics::dec_gauge_vec(
-                        &metrics::PEERS_PER_CLIENT,
-                        &[&previous_kind.to_string()],
+                        &[peer_info.client().kind.as_ref()],
                     );
+                    metrics::dec_gauge_vec(&metrics::PEERS_PER_CLIENT, &[previous_kind.as_ref()]);
                 }
             }
         } else {
@@ -674,7 +671,7 @@ impl<TSpec: EthSpec> PeerManager<TSpec> {
             let value = clients_per_peer.get(&client_kind.to_string()).unwrap_or(&0);
             metrics::set_gauge_vec(
                 &metrics::PEERS_PER_CLIENT,
-                &[&client_kind.to_string()],
+                &[client_kind.as_ref()],
                 *value as i64,
             );
         }

diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb/client.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb/client.rs
@@ -4,7 +4,7 @@
 
 use libp2p::identify::IdentifyInfo;
 use serde::Serialize;
-use strum::{AsRefStr, AsStaticStr, EnumIter};
+use strum::{AsRefStr, EnumIter, IntoStaticStr};
 
 /// Various client and protocol information related to a node.
 #[derive(Clone, Debug, Serialize)]
@@ -21,7 +21,7 @@ pub struct Client {
     pub agent_string: Option<String>,
 }
 
-#[derive(Clone, Debug, Serialize, PartialEq, AsRefStr, AsStaticStr, EnumIter)]
+#[derive(Clone, Copy, Debug, Serialize, PartialEq, AsRefStr, IntoStaticStr, EnumIter)]
 pub enum ClientKind {
     /// A lighthouse node (the best kind).
     Lighthouse,

diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs
@@ -9,7 +9,7 @@ use ssz_types::{
     VariableList,
 };
 use std::ops::Deref;
-use strum::AsStaticStr;
+use strum::IntoStaticStr;
 use superstruct::superstruct;
 use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot};
 
@@ -263,7 +263,7 @@ pub enum RPCCodedResponse<T: EthSpec> {
 }
 
 /// The code assigned to an erroneous `RPCResponse`.
-#[derive(Debug, Clone, Copy, PartialEq, AsStaticStr)]
+#[derive(Debug, Clone, Copy, PartialEq, IntoStaticStr)]
 #[strum(serialize_all = "snake_case")]
 pub enum RPCResponseErrorCode {
     RateLimited,

diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs
@@ -14,7 +14,7 @@ use std::io;
 use std::marker::PhantomData;
 use std::sync::Arc;
 use std::time::Duration;
-use strum::{AsStaticRef, AsStaticStr};
+use strum::IntoStaticStr;
 use tokio_io_timeout::TimeoutStream;
 use tokio_util::{
     codec::Framed,
@@ -510,7 +510,7 @@ impl<TSpec: EthSpec> InboundRequest<TSpec> {
 }
 
 /// Error in RPC Encoding/Decoding.
-#[derive(Debug, Clone, PartialEq, AsStaticStr)]
+#[derive(Debug, Clone, PartialEq, IntoStaticStr)]
 #[strum(serialize_all = "snake_case")]
 pub enum RPCError {
     /// Error when decoding the raw buffer from ssz.
@@ -617,8 +617,8 @@ impl RPCError {
     /// Used for metrics.
     pub fn as_static_str(&self) -> &'static str {
         match self {
-            RPCError::ErrorResponse(ref code, ..) => code.as_static(),
-            e => e.as_static(),
+            RPCError::ErrorResponse(ref code, ..) => code.into(),
+            e => e.into(),
         }
     }
 }
diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml
@@ -41,5 +41,5 @@ itertools = "0.10.0"
 num_cpus = "1.13.0"
 lru_cache = { path = "../../common/lru_cache" }
 if-addrs = "0.6.4"
-strum = "0.21.0"
+strum = "0.24.0"
 tokio-util = { version = "0.6.3", features = ["time"] }
diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs
@@ -9,7 +9,6 @@ use lighthouse_network::{
     Gossipsub, NetworkGlobals,
 };
 use std::sync::Arc;
-use strum::AsStaticRef;
 use strum::IntoEnumIterator;
 use types::EthSpec;
 
@@ -357,12 +356,12 @@ pub fn update_gossip_metrics<T: EthSpec>(
     for client_kind in ClientKind::iter() {
         set_gauge_vec(
             &BEACON_BLOCK_MESH_PEERS_PER_CLIENT,
-            &[&client_kind.to_string()],
+            &[client_kind.as_ref()],
             0_i64,
         );
         set_gauge_vec(
             &BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,
-            &[&client_kind.to_string()],
+            &[client_kind.as_ref()],
             0_i64,
         );
     }
@@ -377,7 +376,7 @@ pub fn update_gossip_metrics<T: EthSpec>(
                             .peers
                             .read()
                             .peer_info(peer_id)
-                            .map(|peer_info| peer_info.client().kind.as_static())
+                            .map(|peer_info| peer_info.client().kind.into())
                             .unwrap_or_else(|| "Unknown");
                         if let Some(v) =
                             get_int_gauge(&BEACON_BLOCK_MESH_PEERS_PER_CLIENT, &[client])
@@ -392,7 +391,7 @@ pub fn update_gossip_metrics<T: EthSpec>(
                             .peers
                             .read()
                             .peer_info(peer_id)
-                            .map(|peer_info| peer_info.client().kind.as_static())
+                            .map(|peer_info| peer_info.client().kind.into())
                             .unwrap_or_else(|| "Unknown");
                         if let Some(v) = get_int_gauge(
                             &BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,

diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs
@@ -8,7 +8,6 @@ use lru_cache::LRUCache;
 use slog::{crit, debug, error, trace, warn, Logger};
 use smallvec::SmallVec;
 use store::{Hash256, SignedBeaconBlock};
-use strum::AsStaticRef;
 use tokio::sync::mpsc;
 
 use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent};
@@ -176,7 +175,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                 // request finished correctly, it will be removed after the block is processed.
             }
             Err(error) => {
-                let msg: &str = error.as_static();
+                let msg: &str = error.into();
                 cx.report_peer(peer_id, PeerAction::LowToleranceError, msg);
                 // Remove the request, if it can be retried it will be added with a new id.
                 let mut req = request.remove();
@@ -243,7 +242,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                 VerifyError::RootMismatch
                 | VerifyError::NoBlockReturned
                 | VerifyError::ExtraBlocksReturned => {
-                    let e = e.as_static();
+                    let e = e.into();
                     warn!(self.log, "Peer sent invalid response to parent request.";
                         "peer_id" => %peer_id, "reason" => e);
 
@@ -310,8 +309,13 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
                     }
                 }
                 Err(e) => {
-                    trace!(self.log, "Single block request failed on peer disconnection";
-                        "block_root" => %req.hash, "peer_id" => %peer_id, "reason" => e.as_static());
+                    trace!(
+                        self.log,
+                        "Single block request failed on peer disconnection";
+                        "block_root" => %req.hash,
+                        "peer_id" => %peer_id,
+                        "reason" => <&str>::from(e),
+                    );
                 }
             }
         }

diff --git a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs
@@ -1,6 +1,6 @@
 use lighthouse_network::PeerId;
 use store::{EthSpec, Hash256, SignedBeaconBlock};
-use strum::AsStaticStr;
+use strum::IntoStaticStr;
 
 use crate::sync::{
     manager::{Id, SLOT_IMPORT_TOLERANCE},
@@ -28,7 +28,7 @@ pub(crate) struct ParentLookup<T: EthSpec> {
     current_parent_request_id: Option<Id>,
 }
 
-#[derive(Debug, PartialEq, Eq, AsStaticStr)]
+#[derive(Debug, PartialEq, Eq, IntoStaticStr)]
 pub enum VerifyError {
     RootMismatch,
     NoBlockReturned,

diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs
@@ -4,7 +4,7 @@ use lighthouse_network::{rpc::BlocksByRootRequest, PeerId};
 use rand::seq::IteratorRandom;
 use ssz_types::VariableList;
 use store::{EthSpec, Hash256, SignedBeaconBlock};
-use strum::AsStaticStr;
+use strum::IntoStaticStr;
 
 /// Object representing a single block lookup request.
 #[derive(PartialEq, Eq)]
@@ -28,14 +28,14 @@ pub enum State {
     Processing { peer_id: PeerId },
 }
 
-#[derive(Debug, PartialEq, Eq, AsStaticStr)]
+#[derive(Debug, PartialEq, Eq, IntoStaticStr)]
 pub enum VerifyError {
     RootMismatch,
     NoBlockReturned,
     ExtraBlocksReturned,
 }
 
-#[derive(Debug, PartialEq, Eq, AsStaticStr)]
+#[derive(Debug, PartialEq, Eq, IntoStaticStr)]
 pub enum LookupRequestError {
     TooManyAttempts,
     NoPeers,

diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs
@@ -284,16 +284,7 @@ pub fn get_config<E: EthSpec>(
         client_config.freezer_db_path = Some(PathBuf::from(freezer_dir));
     }
 
-    if let Some(slots_per_restore_point) = cli_args.value_of("slots-per-restore-point") {
-        client_config.store.slots_per_restore_point = slots_per_restore_point
-            .parse()
-            .map_err(|_| "slots-per-restore-point is not a valid integer".to_string())?;
-    } else {
-        client_config.store.slots_per_restore_point = std::cmp::min(
-            E::slots_per_historical_root() as u64,
-            store::config::DEFAULT_SLOTS_PER_RESTORE_POINT,
-        );
-    }
+    client_config.store.slots_per_restore_point = get_slots_per_restore_point::<E>(cli_args)?;
 
     if let Some(block_cache_size) = cli_args.value_of("block-cache-size") {
         client_config.store.block_cache_size = block_cache_size
@@ -820,3 +811,17 @@ pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf {
         })
         .unwrap_or_else(|| PathBuf::from("."))
 }
+
+/// Get the `slots_per_restore_point` value to use for the database.
+pub fn get_slots_per_restore_point<E: EthSpec>(cli_args: &ArgMatches) -> Result<u64, String> {
+    if let Some(slots_per_restore_point) =
+        clap_utils::parse_optional(cli_args, "slots-per-restore-point")?
+    {
+        Ok(slots_per_restore_point)
+    } else {
+        Ok(std::cmp::min(
+            E::slots_per_historical_root() as u64,
+            store::config::DEFAULT_SLOTS_PER_RESTORE_POINT,
+        ))
+    }
+}
diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs
@@ -13,7 +13,7 @@ use beacon_chain::{
 use clap::ArgMatches;
 pub use cli::cli_app;
 pub use client::{Client, ClientBuilder, ClientConfig, ClientGenesis};
-pub use config::{get_config, get_data_dir, set_network_config};
+pub use config::{get_config, get_data_dir, get_slots_per_restore_point, set_network_config};
 use environment::RuntimeContext;
 pub use eth2_config::Eth2Config;
 use slasher::Slasher;