From 5d98ce117c5df67f1d0a90bea073562320782586 Mon Sep 17 00:00:00 2001 From: Eran Rundstein Date: Wed, 28 Jun 2023 08:44:45 -0700 Subject: [PATCH] light-client-cli: add command to fetch archive blocks and verify them (#3399) * add fetch archive blocks subcommand * add verification support * fix dependency issue and add test * use clio for output file argument * fix typo * lint --- Cargo.lock | 30 +++- ledger/sync/Cargo.toml | 3 +- light-client/cli/Cargo.toml | 4 + light-client/cli/src/bin/main.rs | 227 +++++++++++++++++++------- light-client/verifier/src/error.rs | 4 + light-client/verifier/src/verifier.rs | 122 +++++++++++++- 6 files changed, 320 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7df58e6a4e..149a5650c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -674,14 +674,14 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.22" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.2", "indexmap", - "textwrap 0.15.1", + "textwrap 0.16.0", ] [[package]] @@ -744,6 +744,20 @@ dependencies = [ "cc", ] +[[package]] +name = "clio" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f8d4adf1833f1d75ee8b5af282fe742de6631f6741d1b02656c3c5c46b7206" +dependencies = [ + "cfg-if 1.0.0", + "clap 4.1.14", + "libc", + "tempfile", + "walkdir", + "windows-sys 0.42.0", +] + [[package]] name = "cmake" version = "0.1.49" @@ -865,7 +879,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.22", + "clap 3.2.25", "criterion-plot", "itertools", "lazy_static", @@ -4846,14 +4860,18 @@ name = "mc-light-client-cli" version = "4.1.0-pre0" dependencies = [ "clap 4.1.14", + "clio", "grpcio", "mc-api", + "mc-blockchain-types", "mc-common", "mc-consensus-api", "mc-consensus-scp-types", + "mc-ledger-sync", "mc-light-client-verifier", "mc-util-grpc", "mc-util-uri", + "protobuf", "serde_json", ] @@ -8085,9 +8103,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" diff --git a/ledger/sync/Cargo.toml b/ledger/sync/Cargo.toml index df6fb9e7e9..384f395753 100644 --- a/ledger/sync/Cargo.toml +++ b/ledger/sync/Cargo.toml @@ -10,6 +10,7 @@ rust-version = { workspace = true } [[bin]] name = "ledger-sync-test-app" path = "src/test_app/main.rs" +required-features = ["mc-consensus-enclave-measurement"] [dependencies] mc-account-keys = { path = "../../account-keys" } @@ -19,7 +20,7 @@ mc-blockchain-test-utils = { path = "../../blockchain/test-utils" } mc-blockchain-types = { path = "../../blockchain/types" } mc-common = { path = "../../common", features = ["log"] } mc-connection = { path = "../../connection" } -mc-consensus-enclave-measurement = { path = "../../consensus/enclave/measurement" } +mc-consensus-enclave-measurement = { path = "../../consensus/enclave/measurement", optional = true } mc-consensus-scp = { path = "../../consensus/scp" } mc-ledger-db = { path = "../../ledger/db" } mc-transaction-core = { path = "../../transaction/core" } diff --git a/light-client/cli/Cargo.toml b/light-client/cli/Cargo.toml index 50fed0eab3..acdc4aaa82 100644 --- a/light-client/cli/Cargo.toml +++ b/light-client/cli/Cargo.toml @@ -12,13 +12,17 @@ path = "src/bin/main.rs" [dependencies] mc-api = { path = "../../api" } +mc-blockchain-types = { path = "../../blockchain/types" } mc-common = { path = "../../common", features = ["log"] } mc-consensus-api = { path = "../../consensus/api" } mc-consensus-scp-types = { path = "../../consensus/scp/types" } +mc-ledger-sync = { path = "../../ledger/sync" } mc-light-client-verifier = { path = "../verifier" } mc-util-grpc = { path = "../../util/grpc" } mc-util-uri = { path = "../../util/uri" } clap = { version = "4.1", features = ["derive", "env"] } +clio = { version = "0.3.1", features = ["clap-parse"] } grpcio = "0.12.1" +protobuf = "2.27.1" serde_json = "1.0" diff --git a/light-client/cli/src/bin/main.rs b/light-client/cli/src/bin/main.rs index af6653c29c..a7fecd1f10 100644 --- a/light-client/cli/src/bin/main.rs +++ b/light-client/cli/src/bin/main.rs @@ -1,27 +1,66 @@ // Copyright (c) 2018-2023 The MobileCoin Foundation use clap::{Parser, Subcommand}; +use clio::Output; use grpcio::{ChannelBuilder, EnvBuilder}; +use mc_api::blockchain::ArchiveBlocks; +use mc_blockchain_types::BlockIndex; use mc_common::{ - logger::{create_app_logger, o}, + logger::{create_app_logger, log, o, Logger}, ResponderId, }; use mc_consensus_api::{ consensus_client_grpc::ConsensusClientApiClient, consensus_common_grpc::BlockchainApiClient, }; use mc_consensus_scp_types::QuorumSet; +use mc_ledger_sync::ReqwestTransactionsFetcher; use mc_light_client_verifier::{ - HexKeyNodeID, LightClientVerifierConfig, TrustedValidatorSetConfig, + HexKeyNodeID, LightClientVerifier, LightClientVerifierConfig, TrustedValidatorSetConfig, }; use mc_util_grpc::ConnectionUriGrpcioChannel; use mc_util_uri::ConsensusClientUri; -use std::{str::FromStr, sync::Arc}; +use protobuf::Message; +use std::{fs, io::Write, path::PathBuf, str::FromStr, sync::Arc}; #[derive(Subcommand)] pub enum Commands { + /// Generate a light client verifier config from a list of nodes. + /// This does not include any historical data. GenerateConfig { + /// Node URIs to use for generating the config. #[clap(long = "node", use_value_delimiter = true, env = "MC_NODES")] nodes: Vec, + + /// File to write the config to. + #[clap(long, env = "MC_OUT_FILE", value_parser, default_value = "-")] + out_file: Output, + }, + + /// Fetch one or more `[ArchiveBlock]`s from a list of tx source urls and + /// store them in a Protobuf file. + FetchArchiveBlocks { + /// URLs to use for fetching blocks. + /// + /// For example: https://ledger.mobilecoinww.com/node1.prod.mobilecoinww.com + #[clap( + long = "tx-source-url", + use_value_delimiter = true, + env = "MC_TX_SOURCE_URL" + )] + tx_source_urls: Vec, + + /// Block index we are interested in. + #[clap(long, env = "MC_BLOCK_INDEX")] + block_index: BlockIndex, + + /// File to write the fetched ArchiveBlocks protobuf to. + #[clap(long, env = "MC_OUT_FILE", value_parser)] + out_file: Output, + + /// Optional LightClientVerifierConfig to use for verifying the fetched + /// blocks before writing them to disk. + #[clap(long, env = "MC_LIGHT_CLIENT_VERIFIER_CONFIG")] + light_client_verifier_config: Option, }, } @@ -39,67 +78,131 @@ fn main() { let (logger, _global_logger_guard) = create_app_logger(o!()); let config = Config::parse(); - let env = Arc::new(EnvBuilder::new().name_prefix("light-client-grpc").build()); - match config.command { - Commands::GenerateConfig { nodes } => { - let (node_configs, last_block_infos): (Vec<_>, Vec<_>) = nodes - .iter() - .map(|node_uri| { - // TODO should this use ThickClient and chain-id? - let ch = ChannelBuilder::default_channel_builder(env.clone()) - .connect_to_uri(node_uri, &logger); - - let client_api = ConsensusClientApiClient::new(ch.clone()); - let config = client_api - .get_node_config(&Default::default()) - .expect("get_node_config failed"); - - let blockchain_api = BlockchainApiClient::new(ch); - let last_block_info = blockchain_api - .get_last_block_info(&Default::default()) - .expect("get_last_block_info failed"); - - (config, last_block_info) - }) - .unzip(); - - let node_ids = node_configs - .iter() - .map(|node_config| HexKeyNodeID { - responder_id: ResponderId::from_str(node_config.get_peer_responder_id()) - .unwrap(), - public_key: node_config - .get_scp_message_signing_key() - .try_into() - .unwrap(), - }) - .collect::>(); - - let quorum_set = QuorumSet { - threshold: node_configs.len() as u32, - members: node_ids.into_iter().map(Into::into).collect(), - }; - - let trusted_validator_set = TrustedValidatorSetConfig { quorum_set }; - - let trusted_validator_set_start_block = last_block_infos - .iter() - .map(|last_block_info| last_block_info.index) - .max() - .unwrap_or_default(); - - let light_client_verifier = LightClientVerifierConfig { - trusted_validator_set, - trusted_validator_set_start_block, - historical_validator_sets: Default::default(), - known_valid_block_ids: Default::default(), - }; - - println!( - "{}", - serde_json::to_string_pretty(&light_client_verifier).unwrap() + Commands::GenerateConfig { nodes, out_file } => { + cmd_generate_config(nodes, out_file, logger); + } + + Commands::FetchArchiveBlocks { + tx_source_urls, + block_index, + out_file, + light_client_verifier_config, + } => { + cmd_fetch_archive_blocks( + tx_source_urls, + block_index, + out_file, + light_client_verifier_config, + logger, ); } } } + +fn cmd_generate_config(nodes: Vec, mut out_file: Output, logger: Logger) { + let env = Arc::new(EnvBuilder::new().name_prefix("light-client-grpc").build()); + + let (node_configs, last_block_infos): (Vec<_>, Vec<_>) = nodes + .iter() + .map(|node_uri| { + // TODO should this use ThickClient and chain-id? + let ch = ChannelBuilder::default_channel_builder(env.clone()) + .connect_to_uri(node_uri, &logger); + + let client_api = ConsensusClientApiClient::new(ch.clone()); + let config = client_api + .get_node_config(&Default::default()) + .expect("get_node_config failed"); + + let blockchain_api = BlockchainApiClient::new(ch); + let last_block_info = blockchain_api + .get_last_block_info(&Default::default()) + .expect("get_last_block_info failed"); + + (config, last_block_info) + }) + .unzip(); + + let node_ids = node_configs + .iter() + .map(|node_config| HexKeyNodeID { + responder_id: ResponderId::from_str(node_config.get_peer_responder_id()).unwrap(), + public_key: node_config + .get_scp_message_signing_key() + .try_into() + .unwrap(), + }) + .collect::>(); + + let quorum_set = QuorumSet { + threshold: node_configs.len() as u32, + members: node_ids.into_iter().map(Into::into).collect(), + }; + + let trusted_validator_set = TrustedValidatorSetConfig { quorum_set }; + + let trusted_validator_set_start_block = last_block_infos + .iter() + .map(|last_block_info| last_block_info.index) + .max() + .unwrap_or_default(); + + let light_client_verifier = LightClientVerifierConfig { + trusted_validator_set, + trusted_validator_set_start_block, + historical_validator_sets: Default::default(), + known_valid_block_ids: Default::default(), + }; + + out_file + .write_all( + serde_json::to_string_pretty(&light_client_verifier) + .unwrap() + .as_bytes(), + ) + .expect("failed writing config to file"); +} + +fn cmd_fetch_archive_blocks( + tx_source_urls: Vec, + block_index: u64, + mut out_file: Output, + light_client_verifier_config_path: Option, + logger: Logger, +) { + let block_data = tx_source_urls + .into_iter() + .map(|url| { + log::info!(logger, "Fetching block data from {}", url); + let rts = ReqwestTransactionsFetcher::new(vec![url], logger.clone()) + .expect("failed creating ReqwestTransactionsFetcher"); + rts.get_block_data_by_index(block_index, None) + .expect("failed fetching block data") + }) + .collect::>(); + + if let Some(path) = light_client_verifier_config_path { + let json_data = + fs::read_to_string(path).expect("failed reading LightClientVerifierConfig file"); + let light_client_verifier_config: LightClientVerifierConfig = + serde_json::from_str(&json_data).expect("failed parsing LightClientVerifierConfig"); + let light_client_verifier = LightClientVerifier::from(light_client_verifier_config); + + light_client_verifier + .verify_block_data(&block_data[..]) + .expect("failed verifying block data"); + } + + let archive_blocks = ArchiveBlocks::from(&block_data[..]); + let bytes = archive_blocks + .write_to_bytes() + .expect("failed serializing ArchiveBlocks"); + out_file + .write_all(&bytes) + .expect("failed writing ArchiveBlocks to file"); + log::info!(logger, "Wrote ArchiveBlocks to file {}", out_file.path()); + + // Give the logger time to flush :/ + std::thread::sleep(std::time::Duration::from_millis(100)); +} diff --git a/light-client/verifier/src/error.rs b/light-client/verifier/src/error.rs index 960453d317..8be006acfc 100644 --- a/light-client/verifier/src/error.rs +++ b/light-client/verifier/src/error.rs @@ -19,4 +19,8 @@ pub enum Error { BlockContentHashMismatch(BlockContentsHash), /// TxOut (public key {0:?}) was not found among the block contents TxOutNotFound([u8; 32]), + /// Not all BlockDatas point at the same block + BlockDataMismatch, + /// No block data was provided + NoBlockData, } diff --git a/light-client/verifier/src/verifier.rs b/light-client/verifier/src/verifier.rs index ed2839eeb0..5aa01ea5e9 100644 --- a/light-client/verifier/src/verifier.rs +++ b/light-client/verifier/src/verifier.rs @@ -1,7 +1,7 @@ // Copyright (c) 2018-2023 The MobileCoin Foundation use crate::{Error, TrustedValidatorSet}; -use mc_blockchain_types::{Block, BlockContents, BlockID, BlockIndex, BlockMetadata}; +use mc_blockchain_types::{Block, BlockContents, BlockData, BlockID, BlockIndex, BlockMetadata}; use mc_transaction_core::tx::TxOut; use serde::{Deserialize, Serialize}; use std::{collections::BTreeSet, ops::Range}; @@ -114,6 +114,37 @@ impl LightClientVerifier { } Ok(()) } + + /// Verify that a list of BlockDatas all contain the same block and + /// block_contents, and that the block was externalized given evidence in + /// the BlockMetadata available. + pub fn verify_block_data( + &self, + block_data: &[BlockData], + ) -> Result<(Block, BlockContents), Error> { + if block_data.is_empty() { + return Err(Error::NoBlockData); + } + + // All block_data should point at the same block/block_contents. + if !block_data + .windows(2) + .all(|w| w[0].block() == w[1].block() && w[0].contents() == w[1].contents()) + { + return Err(Error::BlockDataMismatch); + } + + let block = block_data[0].block(); + let block_contents = block_data[0].contents(); + + let block_metadata = block_data + .iter() + .filter_map(|block_data| block_data.metadata().cloned()) + .collect::>(); + self.verify_block_and_block_contents(block, block_contents, &block_metadata[..])?; + + Ok((block.clone(), block_contents.clone())) + } } #[cfg(test)] @@ -382,4 +413,93 @@ mod tests { drop(txo2); drop(txo3); } + + #[test] + fn test_verify_block_data() { + let mut rng = get_seeded_rng(); + + let lcv = get_light_client_verifier(Default::default()); + + let txo1 = TxOut::new( + Default::default(), + Amount::new(1, 0.into()), + &FromRandom::from_random(&mut rng), + &FromRandom::from_random(&mut rng), + EncryptedFogHint::fake_onetime_hint(&mut rng), + ) + .unwrap(); + + let txo2 = TxOut::new( + Default::default(), + Amount::new(2, 0.into()), + &FromRandom::from_random(&mut rng), + &FromRandom::from_random(&mut rng), + EncryptedFogHint::fake_onetime_hint(&mut rng), + ) + .unwrap(); + + let txo3 = TxOut::new( + Default::default(), + Amount::new(2, 0.into()), + &FromRandom::from_random(&mut rng), + &FromRandom::from_random(&mut rng), + EncryptedFogHint::fake_onetime_hint(&mut rng), + ) + .unwrap(); + + let bc = BlockContents { + outputs: vec![txo1, txo2], + ..Default::default() + }; + + let block9999 = Block::new( + Default::default(), + &Default::default(), + 9999, + 9999, + &Default::default(), + &bc, + ); + + let metadata = sign_block_id_for_test_node_ids(&block9999.id, &[1, 2, 3]); + + let block_datas = metadata + .into_iter() + .map(|metadata| BlockData::new(block9999.clone(), bc.clone(), None, Some(metadata))) + .collect::>(); + + // Calling verify_block_data without any block data returns an error. + assert_matches!(lcv.verify_block_data(&[]), Err(Error::NoBlockData)); + + // Calling verify_block_data with block data that differs from each other + // returns an error. + let different_block_data = block_datas[0].clone(); + let different_block_data = + different_block_data.mutate(|_, contents, _, _| contents.outputs.push(txo3)); + + assert_matches!( + lcv.verify_block_data(&[block_datas[0].clone(), different_block_data]), + Err(Error::BlockDataMismatch) + ); + + // Passing the correct block metadata verifies successfully and returns the + // block and its contents. + let (block, block_contents) = lcv.verify_block_data(&block_datas[..]).unwrap(); + + assert_eq!(block, block9999); + assert_eq!(block_contents, bc); + + // Omitting one of the block data entries should still succeed since we need 2 + // out of 3. + let (block, block_contents) = lcv.verify_block_data(&block_datas[1..]).unwrap(); + + assert_eq!(block, block9999); + assert_eq!(block_contents, bc); + + // Omitting two of the block data entries should fail since we need 2 out of 3. + assert_matches!( + lcv.verify_block_data(&block_datas[2..]), + Err(Error::NotAQuorum) + ); + } }