Skip to content

Commit

Permalink
Merge branch 'main' into test-registerkeys-service
Browse files Browse the repository at this point in the history
  • Loading branch information
upbqdn committed Feb 16, 2024
2 parents f696609 + fbddec3 commit a9ff9a7
Show file tree
Hide file tree
Showing 18 changed files with 316 additions and 21 deletions.
5 changes: 5 additions & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -5836,13 +5836,18 @@ version = "0.1.0-alpha.1"
dependencies = [
"color-eyre",
"futures-util",
"insta",
"prost",
"serde",
"tokio",
"tonic 0.11.0",
"tonic-build 0.11.0",
"tower",
"zcash_primitives",
"zebra-chain",
"zebra-node-services",
"zebra-state",
"zebra-test",
]

[[package]]
Expand Down
4 changes: 2 additions & 2 deletions book/src/user/supported-platforms.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ For the full requirements, see [Tier 1 platform policy](platform-tier-policy.md#

| platform | os | notes | rust | artifacts
| -------|-------|-------|-------|-------
| `x86_64-unknown-linux-gnu` | [Debian 11](https://www.debian.org/releases/bullseye/) | 64-bit | [latest stable release](https://github.com/rust-lang/rust/releases) | Docker
| `x86_64-unknown-linux-gnu` | [Debian 11](https://www.debian.org/releases/bookworm/) | 64-bit | [latest stable release](https://github.com/rust-lang/rust/releases) | Docker

## Tier 2

Expand Down Expand Up @@ -46,5 +46,5 @@ For the full requirements, see [Tier 3 platform policy](platform-tier-policy.md#

| platform | os | notes | rust | artifacts
| -------|-------|-------|-------|-------
| `aarch64-unknown-linux-gnu` | [Debian 11](https://www.debian.org/releases/bullseye/) | 64-bit | [latest stable release](https://github.com/rust-lang/rust/releases) | N/A
| `aarch64-unknown-linux-gnu` | [Debian 11](https://www.debian.org/releases/bookworm/) | 64-bit | [latest stable release](https://github.com/rust-lang/rust/releases) | N/A
| `aarch64-apple-darwin` | latest macOS | 64-bit, Apple M1 or M2 | [latest stable release](https://github.com/rust-lang/rust/releases) | N/A
2 changes: 1 addition & 1 deletion book/src/user/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ for:
- macOS,
- Ubuntu,
- Docker:
- Debian Bullseye.
- Debian Bookworm.

## Memory Issues

Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints"
ARG EXPERIMENTAL_FEATURES=""

# This stage implements cargo-chef for docker layer caching
FROM rust:bullseye as chef
FROM rust:bookworm as chef
RUN cargo install cargo-chef --locked
WORKDIR /opt/zebrad

Expand Down Expand Up @@ -181,7 +181,7 @@ RUN chmod u+x /entrypoint.sh
#
# To save space, this step starts from scratch using debian, and only adds the resulting
# binary from the `release` stage
FROM debian:bullseye-slim AS runtime
FROM debian:bookworm-slim AS runtime
COPY --from=release /opt/zebrad/target/release/zebrad /usr/local/bin
COPY --from=release /entrypoint.sh /

Expand Down
2 changes: 1 addition & 1 deletion docker/zcash-lightwalletd/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ CMD ["--no-tls-very-insecure", "--grpc-bind-addr=0.0.0.0:9067", "--http-bind-ad
##
## Deploy
##
FROM debian:bullseye-slim as runtime
FROM debian:bookworm-slim as runtime

ARG ZCASHD_CONF_PATH
# Maintain backward compatibility with mainstream repo using this ARGs in docker-compose
Expand Down
9 changes: 9 additions & 0 deletions zebra-grpc/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ categories = ["cryptography::cryptocurrencies"]
futures-util = "0.3.28"
tonic = "0.11.0"
prost = "0.12.3"
serde = { version = "1.0.196", features = ["serde_derive"] }
tokio = { version = "1.36.0", features = ["macros", "rt-multi-thread"] }
tower = { version = "0.4.13", features = ["util", "buffer"] }
color-eyre = "0.6.2"
Expand All @@ -29,3 +30,11 @@ zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.3

[build-dependencies]
tonic-build = "0.11.0"

[dev-dependencies]
insta = { version = "1.33.0", features = ["redactions", "json", "ron"] }

zebra-chain = { path = "../zebra-chain", features = ["proptest-impl"] }
zebra-state = { path = "../zebra-state" }
zebra-test = { path = "../zebra-test" }

1 change: 1 addition & 0 deletions zebra-grpc/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
fn main() -> Result<(), Box<dyn std::error::Error>> {
tonic_build::configure()
.btree_map(["."])
.type_attribute(".", "#[derive(serde::Deserialize, serde::Serialize)]")
.compile(&["proto/scanner.proto"], &[""])?;
Ok(())
}
23 changes: 23 additions & 0 deletions zebra-grpc/proto/scanner.proto
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ service Scanner {

// Get all data we have stored for the given keys.
rpc GetResults(GetResultsRequest) returns (GetResultsResponse);

// Submits scanning keys to the scanner.
rpc RegisterKeys(RegisterKeysRequest) returns (RegisterKeysResponse);
}

// A response to a GetInfo call.
Expand All @@ -45,12 +48,24 @@ message GetResultsRequest {
repeated string keys = 1;
}

// A request to register scanning keys
message RegisterKeysRequest {
// Keys to register
repeated KeyWithHeight keys = 1;
}

// A set of responses for each provided key of a GetResults call.
message GetResultsResponse {
// Results for each key.
map<string, Results> results = 1;
}

// A response to `RegisterKeysRequest` containing registered keys
message RegisterKeysResponse {
// Keys that were registered
repeated string keys = 1;
}

// A result for a single key.
message Results {
// A height, transaction id map
Expand All @@ -62,3 +77,11 @@ message TransactionHash {
// A transaction id hash
repeated string hash = 1;
}

// A scanning key with an optional birth height
message KeyWithHeight {
// Scanning key
string key = 1;
// Birth height of the key
optional uint32 height = 2;
}
3 changes: 3 additions & 0 deletions zebra-grpc/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@

pub mod server;

#[cfg(test)]
mod tests;

/// The generated scanner proto
pub mod scanner {
tonic::include_proto!("scanner");
Expand Down
42 changes: 33 additions & 9 deletions zebra-grpc/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use std::{collections::BTreeMap, net::SocketAddr};

use futures_util::future::TryFutureExt;
use tonic::{transport::Server, Response, Status};
use tonic::{transport::Server, Request, Response, Status};
use tower::ServiceExt;

use zebra_node_services::scan_service::{
Expand All @@ -13,7 +13,7 @@ use zebra_node_services::scan_service::{
use crate::scanner::{
scanner_server::{Scanner, ScannerServer},
ClearResultsRequest, DeleteKeysRequest, Empty, GetResultsRequest, GetResultsResponse,
InfoReply, Results, TransactionHash,
InfoReply, RegisterKeysRequest, RegisterKeysResponse, Results, TransactionHash,
};

type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
Expand Down Expand Up @@ -42,10 +42,7 @@ where
+ 'static,
<ScanService as tower::Service<ScanServiceRequest>>::Future: Send,
{
async fn get_info(
&self,
_request: tonic::Request<Empty>,
) -> Result<Response<InfoReply>, Status> {
async fn get_info(&self, _request: Request<Empty>) -> Result<Response<InfoReply>, Status> {
let ScanServiceResponse::Info {
min_sapling_birthday_height,
} = self
Expand All @@ -68,9 +65,36 @@ where
Ok(Response::new(reply))
}

async fn register_keys(
&self,
request: Request<RegisterKeysRequest>,
) -> Result<Response<RegisterKeysResponse>, Status> {
let keys = request
.into_inner()
.keys
.into_iter()
.map(|key_with_height| (key_with_height.key, key_with_height.height))
.collect();

let ScanServiceResponse::RegisteredKeys(keys) = self
.scan_service
.clone()
.ready()
.and_then(|service| service.call(ScanServiceRequest::RegisterKeys(keys)))
.await
.map_err(|_| Status::unknown("scan service was unavailable"))?
else {
return Err(Status::unknown(
"scan service returned an unexpected response",
));
};

Ok(Response::new(RegisterKeysResponse { keys }))
}

async fn clear_results(
&self,
request: tonic::Request<ClearResultsRequest>,
request: Request<ClearResultsRequest>,
) -> Result<Response<Empty>, Status> {
let keys = request.into_inner().keys;

Expand All @@ -97,7 +121,7 @@ where

async fn delete_keys(
&self,
request: tonic::Request<DeleteKeysRequest>,
request: Request<DeleteKeysRequest>,
) -> Result<Response<Empty>, Status> {
let keys = request.into_inner().keys;

Expand All @@ -124,7 +148,7 @@ where

async fn get_results(
&self,
request: tonic::Request<GetResultsRequest>,
request: Request<GetResultsRequest>,
) -> Result<Response<GetResultsResponse>, Status> {
let keys = request.into_inner().keys;

Expand Down
1 change: 1 addition & 0 deletions zebra-grpc/src/tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
mod snapshot;
151 changes: 151 additions & 0 deletions zebra-grpc/src/tests/snapshot.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
//! Snapshot tests for Zebra Scan gRPC responses.
//!
//! Currently we snapshot the `get_info` and `get_results` responses for both mainnet and testnet with a
//! mocked scanner database. Calls that return `Empty` responses are not snapshoted in this suite.
//!
//! To update these snapshots, run:
//! ```sh
//! cargo insta test --review --delete-unreferenced-snapshots
//! ```
use std::{collections::BTreeMap, thread::sleep, time::Duration};

use zebra_chain::{block::Height, parameters::Network, transaction};
use zebra_test::mock_service::MockService;

use zebra_node_services::scan_service::{
request::Request as ScanRequest, response::Response as ScanResponse,
};

use crate::{
scanner::{
scanner_client::ScannerClient, Empty, GetResultsRequest, GetResultsResponse, InfoReply,
},
server::init,
};

/// The extended Sapling viewing key of [ZECpages](https://zecpages.com/boardinfo)
pub const ZECPAGES_SAPLING_VIEWING_KEY: &str = "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz";

#[tokio::test(flavor = "multi_thread")]
async fn test_grpc_response_data() {
let _init_guard = zebra_test::init();

tokio::join!(
test_mocked_rpc_response_data_for_network(
Network::Mainnet,
zebra_test::net::random_known_port()
),
test_mocked_rpc_response_data_for_network(
Network::Testnet,
zebra_test::net::random_known_port()
),
);
}

async fn test_mocked_rpc_response_data_for_network(network: Network, random_port: u16) {
// get a mocked scan service
let mock_scan_service = MockService::build().for_unit_tests();

// start the gRPC server
let listen_addr: std::net::SocketAddr = format!("127.0.0.1:{random_port}")
.parse()
.expect("hard-coded IP and u16 port should parse successfully");

{
let mock_scan_service = mock_scan_service.clone();
tokio::spawn(async move {
init(listen_addr, mock_scan_service)
.await
.expect("Possible port conflict");
});
}

// wait for the server to start
sleep(Duration::from_secs(1));

// connect to the gRPC server
let client = ScannerClient::connect(format!("http://127.0.0.1:{random_port}"))
.await
.expect("server should receive connection");

// insta settings
let mut settings = insta::Settings::clone_current();
settings.set_snapshot_suffix(network.lowercase_name());

// snapshot the get_info grpc call
let get_info_response_fut = {
let mut client = client.clone();
let get_info_request = tonic::Request::new(Empty {});
tokio::spawn(async move { client.get_info(get_info_request).await })
};

{
let mut mock_scan_service = mock_scan_service.clone();
tokio::spawn(async move {
mock_scan_service
.expect_request_that(|req| matches!(req, ScanRequest::Info))
.await
.respond(ScanResponse::Info {
min_sapling_birthday_height: network.sapling_activation_height(),
})
});
}

// snapshot the get_info grpc call

let get_info_response = get_info_response_fut
.await
.expect("tokio task should join successfully")
.expect("get_info request should succeed");

snapshot_rpc_getinfo(get_info_response.into_inner(), &settings);

// snapshot the get_results grpc call

let get_results_response_fut = {
let mut client = client.clone();
let get_results_request = tonic::Request::new(GetResultsRequest {
keys: vec![ZECPAGES_SAPLING_VIEWING_KEY.to_string()],
});
tokio::spawn(async move { client.get_results(get_results_request).await })
};

{
let mut mock_scan_service = mock_scan_service.clone();
tokio::spawn(async move {
let zec_pages_sapling_efvk = ZECPAGES_SAPLING_VIEWING_KEY.to_string();
let mut fake_results = BTreeMap::new();
for fake_result_height in [Height::MIN, Height(1), Height::MAX] {
fake_results.insert(
fake_result_height,
[transaction::Hash::from([0; 32])].repeat(3),
);
}

let mut fake_results_response = BTreeMap::new();
fake_results_response.insert(zec_pages_sapling_efvk, fake_results);

mock_scan_service
.expect_request_that(|req| matches!(req, ScanRequest::Results(_)))
.await
.respond(ScanResponse::Results(fake_results_response))
});
}

let get_results_response = get_results_response_fut
.await
.expect("tokio task should join successfully")
.expect("get_results request should succeed");

snapshot_rpc_getresults(get_results_response.into_inner(), &settings);
}

/// Snapshot `getinfo` response, using `cargo insta` and JSON serialization.
fn snapshot_rpc_getinfo(info: InfoReply, settings: &insta::Settings) {
settings.bind(|| insta::assert_json_snapshot!("get_info", info));
}

/// Snapshot `getresults` response, using `cargo insta` and JSON serialization.
fn snapshot_rpc_getresults(results: GetResultsResponse, settings: &insta::Settings) {
settings.bind(|| insta::assert_json_snapshot!("get_results", results));
}
Loading

0 comments on commit a9ff9a7

Please sign in to comment.