Add beacon.watch (sigp#3362)

> This is currently a WIP and all features are subject to alteration or removal at any time.

The successor to sigp#2873.

Contains the backbone of `beacon.watch` including syncing code, the initial API, and several core database tables.

See `watch/README.md` for more information, requirements and usage.

Cargo.toml

@@ -89,6 +89,8 @@ members = [
 
     "validator_client",
     "validator_client/slashing_protection",
+
+    "watch",
 ]
 resolver = "2"
 

beacon_node/http_api/Cargo.toml

@@ -37,15 +37,15 @@ sysinfo = "0.26.5"
 system_health = { path = "../../common/system_health" }
 directory = { path = "../../common/directory" }
 operation_pool = { path = "../operation_pool" }
+sensitive_url = { path = "../../common/sensitive_url" }
+unused_port = {path = "../../common/unused_port"}
+logging = { path = "../../common/logging" }
+store = { path = "../store" }
 
 [dev-dependencies]
-store = { path = "../store" }
 environment = { path = "../../lighthouse/environment" }
-sensitive_url = { path = "../../common/sensitive_url" }
-logging = { path = "../../common/logging" }
 serde_json = "1.0.58"
 proto_array = { path = "../../consensus/proto_array" }
-unused_port = {path = "../../common/unused_port"}
 genesis = { path = "../genesis" }
 
 [[test]]

beacon_node/http_api/src/lib.rs

@@ -19,6 +19,7 @@ mod standard_block_rewards;
 mod state_id;
 mod sync_committee_rewards;
 mod sync_committees;
+pub mod test_utils;
 mod ui;
 mod validator_inclusion;
 mod version;

beacon_node/http_api/tests/common.rs → beacon_node/http_api/src/test_utils.rs

@@ -1,10 +1,10 @@
+use crate::{Config, Context};
 use beacon_chain::{
     test_utils::{BeaconChainHarness, BoxedMutator, Builder, EphemeralHarnessType},
     BeaconChain, BeaconChainTypes,
 };
 use directory::DEFAULT_ROOT_DIR;
 use eth2::{BeaconNodeHttpClient, Timeouts};
-use http_api::{Config, Context};
 use lighthouse_network::{
     discv5::enr::{CombinedKey, EnrBuilder},
     libp2p::{
@@ -179,7 +179,7 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
     let eth1_service =
         eth1::Service::new(eth1::Config::default(), log.clone(), chain.spec.clone()).unwrap();
 
-    let context = Arc::new(Context {
+    let ctx = Arc::new(Context {
         config: Config {
             enabled: true,
             listen_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)),
@@ -190,19 +190,19 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
             data_dir: std::path::PathBuf::from(DEFAULT_ROOT_DIR),
             spec_fork_name: None,
         },
-        chain: Some(chain.clone()),
+        chain: Some(chain),
         network_senders: Some(network_senders),
         network_globals: Some(network_globals),
         eth1_service: Some(eth1_service),
         log,
     });
-    let ctx = context.clone();
+
     let (shutdown_tx, shutdown_rx) = oneshot::channel();
     let server_shutdown = async {
         // It's not really interesting why this triggered, just that it happened.
         let _ = shutdown_rx.await;
     };
-    let (listening_socket, server) = http_api::serve(ctx, server_shutdown).unwrap();
+    let (listening_socket, server) = crate::serve(ctx, server_shutdown).unwrap();
 
     ApiServer {
         server,

beacon_node/http_api/tests/fork_tests.rs

@@ -1,11 +1,11 @@
 //! Tests for API behaviour across fork boundaries.
-use crate::common::*;
 use beacon_chain::{
     test_utils::{RelativeSyncCommittee, DEFAULT_ETH1_BLOCK_HASH, HARNESS_GENESIS_TIME},
     StateSkipConfig,
 };
 use eth2::types::{IndexedErrorMessage, StateId, SyncSubcommittee};
 use genesis::{bls_withdrawal_credentials, interop_genesis_state_with_withdrawal_credentials};
+use http_api::test_utils::*;
 use std::collections::HashSet;
 use types::{
     test_utils::{generate_deterministic_keypair, generate_deterministic_keypairs},

beacon_node/http_api/tests/interactive_tests.rs

@@ -1,11 +1,11 @@
 //! Generic tests that make use of the (newer) `InteractiveApiTester`
-use crate::common::*;
 use beacon_chain::{
     chain_config::ReOrgThreshold,
     test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy},
 };
 use eth2::types::DepositContractData;
 use execution_layer::{ForkchoiceState, PayloadAttributes};
+use http_api::test_utils::InteractiveTester;
 use parking_lot::Mutex;
 use slot_clock::SlotClock;
 use state_processing::{

beacon_node/http_api/tests/main.rs

@@ -1,7 +1,6 @@
 #![cfg(not(debug_assertions))] // Tests are too slow in debug.
 #![recursion_limit = "512"]
 
-pub mod common;
 pub mod fork_tests;
 pub mod interactive_tests;
 pub mod tests;

beacon_node/http_api/tests/tests.rs

@@ -1,4 +1,3 @@
-use crate::common::{create_api_server, create_api_server_on_port, ApiServer};
 use beacon_chain::test_utils::RelativeSyncCommittee;
 use beacon_chain::{
     test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
@@ -18,7 +17,10 @@ use execution_layer::test_utils::{
 };
 use futures::stream::{Stream, StreamExt};
 use futures::FutureExt;
-use http_api::{BlockId, StateId};
+use http_api::{
+    test_utils::{create_api_server, create_api_server_on_port, ApiServer},
+    BlockId, StateId,
+};
 use lighthouse_network::{Enr, EnrExt, PeerId};
 use network::NetworkReceivers;
 use proto_array::ExecutionStatus;

common/eth2/src/lib.rs

@@ -22,7 +22,7 @@ use lighthouse_network::PeerId;
 pub use reqwest;
 use reqwest::{IntoUrl, RequestBuilder, Response};
 pub use reqwest::{StatusCode, Url};
-pub use sensitive_url::SensitiveUrl;
+pub use sensitive_url::{SensitiveError, SensitiveUrl};
 use serde::{de::DeserializeOwned, Serialize};
 use std::convert::TryFrom;
 use std::fmt;

common/eth2/src/lighthouse.rs

@@ -13,7 +13,7 @@ use crate::{
         BeaconState, ChainSpec, DepositTreeSnapshot, Epoch, EthSpec, FinalizedExecutionBlock,
         GenericResponse, ValidatorId,
     },
-    BeaconNodeHttpClient, DepositData, Error, Eth1Data, Hash256, StateId, StatusCode,
+    BeaconNodeHttpClient, DepositData, Error, Eth1Data, Hash256, Slot, StateId, StatusCode,
 };
 use proto_array::core::ProtoArray;
 use reqwest::IntoUrl;
@@ -566,4 +566,73 @@ impl BeaconNodeHttpClient {
 
         self.post_with_response(path, &()).await
     }
+
+    ///
+    /// Analysis endpoints.
+    ///
+
+    /// `GET` lighthouse/analysis/block_rewards?start_slot,end_slot
+    pub async fn get_lighthouse_analysis_block_rewards(
+        &self,
+        start_slot: Slot,
+        end_slot: Slot,
+    ) -> Result<Vec<BlockReward>, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("analysis")
+            .push("block_rewards");
+
+        path.query_pairs_mut()
+            .append_pair("start_slot", &start_slot.to_string())
+            .append_pair("end_slot", &end_slot.to_string());
+
+        self.get(path).await
+    }
+
+    /// `GET` lighthouse/analysis/block_packing?start_epoch,end_epoch
+    pub async fn get_lighthouse_analysis_block_packing(
+        &self,
+        start_epoch: Epoch,
+        end_epoch: Epoch,
+    ) -> Result<Vec<BlockPackingEfficiency>, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("analysis")
+            .push("block_packing_efficiency");
+
+        path.query_pairs_mut()
+            .append_pair("start_epoch", &start_epoch.to_string())
+            .append_pair("end_epoch", &end_epoch.to_string());
+
+        self.get(path).await
+    }
+
+    /// `GET` lighthouse/analysis/attestation_performance/{index}?start_epoch,end_epoch
+    pub async fn get_lighthouse_analysis_attestation_performance(
+        &self,
+        start_epoch: Epoch,
+        end_epoch: Epoch,
+        target: String,
+    ) -> Result<Vec<AttestationPerformance>, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("analysis")
+            .push("attestation_performance")
+            .push(&target);
+
+        path.query_pairs_mut()
+            .append_pair("start_epoch", &start_epoch.to_string())
+            .append_pair("end_epoch", &end_epoch.to_string());
+
+        self.get(path).await
+    }
 }

watch/.gitignore

@@ -0,0 +1 @@
+config.yaml

watch/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "watch"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+name = "watch"
+path = "src/lib.rs"
+
+[[bin]]
+name = "watch"
+path = "src/main.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = "2.33.3"
+log = "0.4.14"
+env_logger = "0.9.0"
+types = { path = "../consensus/types" }
+eth2 = { path = "../common/eth2" }
+beacon_node = { path = "../beacon_node"}
+tokio = { version = "1.14.0", features = ["time"] }
+axum = "0.5.15"
+hyper = "0.14.20"
+serde = "1.0.116"
+serde_json = "1.0.58"
+reqwest = { version = "0.11.0", features = ["json","stream"] }
+url = "2.2.2"
+rand = "0.7.3"
+diesel = { version = "2.0.2", features = ["postgres", "r2d2"] }
+diesel_migrations = { version = "2.0.0", features = ["postgres"] }
+byteorder = "1.4.3"
+bls = { path = "../crypto/bls" }
+hex = "0.4.2"
+r2d2 = "0.8.9"
+serde_yaml = "0.8.24"
+
+[dev-dependencies]
+tokio-postgres = "0.7.5"
+http_api = { path = "../beacon_node/http_api" }
+beacon_chain = { path = "../beacon_node/beacon_chain" }
+network = { path = "../beacon_node/network" }
+testcontainers = "0.14.0"
+unused_port = { path = "../common/unused_port" }