From ce8acfb2ad606da6f78f53fdcac92793ad139b42 Mon Sep 17 00:00:00 2001 From: Nurullah Giray Kuru Date: Mon, 7 Aug 2023 15:25:15 -0700 Subject: [PATCH] [api] Add individual step timing to api tester (#9502) This commit builds individual step timing on top of the consistent API testing introduced here. See proposal for more high level information. This addition allows us to dive deeper on what the issue is should we detect any flow is consistently slower. Changes: Add timer macro in macros.rs. Add helpers for emitting metrics and refactor process_result. Put publish_module in its own thread by creating a tokio runtime. Add sleeps between persistent checks. Reduce API client sleep time to 100ms from 500ms. Make test setups persistent. --- crates/aptos-api-tester/src/consts.rs | 55 ++++ crates/aptos-api-tester/src/counters.rs | 19 ++ crates/aptos-api-tester/src/fail_message.rs | 50 ++-- crates/aptos-api-tester/src/macros.rs | 18 ++ crates/aptos-api-tester/src/main.rs | 129 +++------- .../aptos-api-tester/src/persistent_check.rs | 57 ++++- .../src/tests/coin_transfer.rs | 104 ++++++-- .../aptos-api-tester/src/tests/new_account.rs | 92 ++++--- .../src/tests/nft_transfer.rs | 171 ++++++++++--- .../src/tests/publish_module.rs | 108 ++++++-- crates/aptos-api-tester/src/utils.rs | 236 +++++++++++++----- crates/aptos-rest-client/src/lib.rs | 3 +- 12 files changed, 746 insertions(+), 296 deletions(-) create mode 100644 crates/aptos-api-tester/src/consts.rs create mode 100644 crates/aptos-api-tester/src/macros.rs diff --git a/crates/aptos-api-tester/src/consts.rs b/crates/aptos-api-tester/src/consts.rs new file mode 100644 index 00000000000000..ef59c196dddc92 --- /dev/null +++ b/crates/aptos-api-tester/src/consts.rs @@ -0,0 +1,55 @@ +// Copyright © Aptos Foundation + +use once_cell::sync::Lazy; +use std::{env, time::Duration}; +use url::Url; + +// faucet constants + +pub static DEVNET_NODE_URL: Lazy = + Lazy::new(|| Url::parse("https://fullnode.devnet.aptoslabs.com").unwrap()); + +pub static DEVNET_FAUCET_URL: Lazy = + Lazy::new(|| Url::parse("https://faucet.devnet.aptoslabs.com").unwrap()); + +pub static TESTNET_NODE_URL: Lazy = + Lazy::new(|| Url::parse("https://fullnode.testnet.aptoslabs.com").unwrap()); + +pub static TESTNET_FAUCET_URL: Lazy = + Lazy::new(|| Url::parse("https://faucet.testnet.aptoslabs.com").unwrap()); + +pub const FUND_AMOUNT: u64 = 100_000_000; + +// persistency check constants + +pub static PERSISTENCY_TIMEOUT: Lazy = Lazy::new(|| { + env::var("PERSISTENCY_TIMEOUT") + .ok() + .and_then(|s| s.parse().ok()) + .map(Duration::from_secs) + .unwrap_or(Duration::from_secs(30)) +}); + +pub static SLEEP_PER_CYCLE: Lazy = Lazy::new(|| { + env::var("SLEEP_PER_CYCLE") + .ok() + .and_then(|s| s.parse().ok()) + .map(Duration::from_millis) + .unwrap_or(Duration::from_millis(100)) +}); + +// runtime constants + +pub static NUM_THREADS: Lazy = Lazy::new(|| { + env::var("NUM_THREADS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(4) +}); + +pub static STACK_SIZE: Lazy = Lazy::new(|| { + env::var("STACK_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(4 * 1024 * 1024) +}); diff --git a/crates/aptos-api-tester/src/counters.rs b/crates/aptos-api-tester/src/counters.rs index 62441749d83a99..1f6305dd644b65 100644 --- a/crates/aptos-api-tester/src/counters.rs +++ b/crates/aptos-api-tester/src/counters.rs @@ -54,3 +54,22 @@ pub static API_TEST_LATENCY: Lazy = Lazy::new(|| { pub fn test_latency(test_name: &str, network_name: &str, run_id: &str, result: &str) -> Histogram { API_TEST_LATENCY.with_label_values(&[test_name, network_name, run_id, result]) } + +pub static API_TEST_STEP_LATENCY: Lazy = Lazy::new(|| { + register_histogram_vec!( + "api_test_step_latency", + "Time it takes to complete a user flow step", + &["test_name", "step_name", "network_name", "run_id", "result"], + ) + .unwrap() +}); + +pub fn test_step_latency( + test_name: &str, + step_name: &str, + network_name: &str, + run_id: &str, + result: &str, +) -> Histogram { + API_TEST_STEP_LATENCY.with_label_values(&[test_name, step_name, network_name, run_id, result]) +} diff --git a/crates/aptos-api-tester/src/fail_message.rs b/crates/aptos-api-tester/src/fail_message.rs index 5e271cc2147caf..9d4065e5050cff 100644 --- a/crates/aptos-api-tester/src/fail_message.rs +++ b/crates/aptos-api-tester/src/fail_message.rs @@ -1,27 +1,27 @@ // Copyright © Aptos Foundation -pub static FAIL_WRONG_ACCOUNT_DATA: &str = "wrong account data"; -pub static FAIL_WRONG_BALANCE: &str = "wrong balance"; -pub static FAIL_WRONG_BALANCE_AT_VERSION: &str = "wrong balance at version"; -pub static FAIL_WRONG_COLLECTION_DATA: &str = "wrong collection data"; -pub static FAIL_WRONG_MESSAGE: &str = "wrong message"; -pub static FAIL_WRONG_MODULE: &str = "wrong module"; -pub static FAIL_WRONG_TOKEN_BALANCE: &str = "wrong token balance"; -pub static FAIL_WRONG_TOKEN_DATA: &str = "wrong token data"; -pub static ERROR_COULD_NOT_BUILD_PACKAGE: &str = "failed to build package"; -pub static ERROR_COULD_NOT_CHECK: &str = "persistency check never started"; -pub static ERROR_COULD_NOT_CREATE_ACCOUNT: &str = "failed to create account"; -pub static ERROR_COULD_NOT_CREATE_TRANSACTION: &str = "failed to create transaction"; -pub static ERROR_COULD_NOT_FINISH_TRANSACTION: &str = "failed to finish transaction"; -pub static ERROR_COULD_NOT_FUND_ACCOUNT: &str = "failed to fund account"; -pub static ERROR_COULD_NOT_SERIALIZE: &str = "failed to serialize"; -pub static ERROR_NO_ACCOUNT_DATA: &str = "can't find account data"; -pub static ERROR_NO_BALANCE: &str = "can't find account balance"; -pub static ERROR_NO_BYTECODE: &str = "can't find bytecode"; -pub static ERROR_NO_COLLECTION_DATA: &str = "can't find collection data"; -pub static ERROR_NO_MESSAGE: &str = "can't find message"; -pub static ERROR_NO_METADATA: &str = "can't find metadata"; -pub static ERROR_NO_MODULE: &str = "can't find module"; -pub static ERROR_NO_TOKEN_BALANCE: &str = "can't find token balance"; -pub static ERROR_NO_TOKEN_DATA: &str = "can't find token data"; -pub static ERROR_NO_VERSION: &str = "can't find transaction version"; +pub const FAIL_WRONG_ACCOUNT_DATA: &str = "wrong account data"; +pub const FAIL_WRONG_BALANCE: &str = "wrong balance"; +pub const FAIL_WRONG_BALANCE_AT_VERSION: &str = "wrong balance at version"; +pub const FAIL_WRONG_COLLECTION_DATA: &str = "wrong collection data"; +pub const FAIL_WRONG_MESSAGE: &str = "wrong message"; +pub const FAIL_WRONG_MODULE: &str = "wrong module"; +pub const FAIL_WRONG_TOKEN_BALANCE: &str = "wrong token balance"; +pub const FAIL_WRONG_TOKEN_DATA: &str = "wrong token data"; +pub const ERROR_COULD_NOT_BUILD_PACKAGE: &str = "failed to build package"; +pub const ERROR_COULD_NOT_CHECK: &str = "persistency check never started"; +pub const ERROR_COULD_NOT_CREATE_ACCOUNT: &str = "failed to create account"; +pub const ERROR_COULD_NOT_CREATE_TRANSACTION: &str = "failed to create transaction"; +pub const ERROR_COULD_NOT_FINISH_TRANSACTION: &str = "failed to finish transaction"; +pub const ERROR_COULD_NOT_FUND_ACCOUNT: &str = "failed to fund account"; +pub const ERROR_COULD_NOT_SERIALIZE: &str = "failed to serialize"; +pub const ERROR_NO_ACCOUNT_DATA: &str = "can't find account data"; +pub const ERROR_NO_BALANCE: &str = "can't find account balance"; +pub const ERROR_NO_BYTECODE: &str = "can't find bytecode"; +pub const ERROR_NO_COLLECTION_DATA: &str = "can't find collection data"; +pub const ERROR_NO_MESSAGE: &str = "can't find message"; +pub const ERROR_NO_METADATA: &str = "can't find metadata"; +pub const ERROR_NO_MODULE: &str = "can't find module"; +pub const ERROR_NO_TOKEN_BALANCE: &str = "can't find token balance"; +pub const ERROR_NO_TOKEN_DATA: &str = "can't find token data"; +pub const ERROR_NO_VERSION: &str = "can't find transaction version"; diff --git a/crates/aptos-api-tester/src/macros.rs b/crates/aptos-api-tester/src/macros.rs new file mode 100644 index 00000000000000..f40d2fd093b58d --- /dev/null +++ b/crates/aptos-api-tester/src/macros.rs @@ -0,0 +1,18 @@ +// Copyright © Aptos Foundation + +#[macro_export] +macro_rules! time_fn { + ($func:expr, $($arg:expr), *) => {{ + // start timer + let start = tokio::time::Instant::now(); + + // call the flow + let result = $func($($arg),+).await; + + // end timer + let time = (tokio::time::Instant::now() - start).as_micros() as f64; + + // return + (result, time) + }}; +} diff --git a/crates/aptos-api-tester/src/main.rs b/crates/aptos-api-tester/src/main.rs index 8a555f112d9165..07079aaadd0e89 100644 --- a/crates/aptos-api-tester/src/main.rs +++ b/crates/aptos-api-tester/src/main.rs @@ -3,135 +3,86 @@ #![forbid(unsafe_code)] +mod consts; mod counters; mod fail_message; mod persistent_check; mod tests; mod utils; +#[macro_use] +mod macros; -use crate::{ - tests::{coin_transfer, new_account, nft_transfer, publish_module}, - utils::{set_metrics, NetworkName, TestFailure, TestName, TestResult}, -}; +use crate::utils::{NetworkName, TestName}; use anyhow::Result; use aptos_logger::{info, Level, Logger}; use aptos_push_metrics::MetricsPusher; +use consts::{NUM_THREADS, STACK_SIZE}; use futures::future::join_all; -use std::{ - future::Future, - time::{Instant, SystemTime, UNIX_EPOCH}, -}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::runtime::{Builder, Runtime}; -// Processes a test result. -async fn process_result>>( - test_name: TestName, - network_name: NetworkName, - run_id: &str, - fut: Fut, -) { - // start timer - let start = Instant::now(); - - // call the flow - let result = fut.await; - - // end timer - let time = (Instant::now() - start).as_micros() as f64; - - // process the result - let output = match result { - Ok(_) => TestResult::Success, - Err(failure) => TestResult::from(failure), - }; - - // set metrics and log - set_metrics( - &output, - &test_name.to_string(), - &network_name.to_string(), - run_id, - time, - ); - info!( - "{} {} result:{:?} in time:{:?}", - network_name.to_string(), - test_name.to_string(), - output, - time, - ); -} - -async fn test_flows(network_name: NetworkName) -> Result<()> { +async fn test_flows(runtime: &Runtime, network_name: NetworkName) -> Result<()> { let run_id = SystemTime::now() .duration_since(UNIX_EPOCH)? .as_secs() .to_string(); - info!("testing {} at {}", network_name.to_string(), run_id); + info!( + "----- STARTING TESTS FOR {} WITH RUN ID {} -----", + network_name.to_string(), + run_id + ); - // Test new account creation and funding + // Flow 1: New account let test_time = run_id.clone(); - let handle_newaccount = tokio::spawn(async move { - process_result( - TestName::NewAccount, - network_name, - &test_time, - new_account::test(network_name), - ) - .await; + let handle_newaccount = runtime.spawn(async move { + TestName::NewAccount.run(network_name, &test_time).await; }); - // Flow 1: Coin transfer + // Flow 2: Coin transfer let test_time = run_id.clone(); - let handle_cointransfer = tokio::spawn(async move { - process_result( - TestName::CoinTransfer, - network_name, - &test_time, - coin_transfer::test(network_name), - ) - .await; + let handle_cointransfer = runtime.spawn(async move { + TestName::CoinTransfer.run(network_name, &test_time).await; }); - // Flow 2: NFT transfer + // Flow 3: NFT transfer let test_time = run_id.clone(); - let handle_nfttransfer = tokio::spawn(async move { - process_result( - TestName::NftTransfer, - network_name, - &test_time, - nft_transfer::test(network_name), - ) - .await; + let handle_nfttransfer = runtime.spawn(async move { + TestName::NftTransfer.run(network_name, &test_time).await; }); - // Flow 3: Publishing module + // Flow 4: Publishing module let test_time = run_id.clone(); - process_result( - TestName::PublishModule, - network_name, - &test_time, - publish_module::test(network_name), - ) - .await; + let handle_publishmodule = runtime.spawn(async move { + TestName::PublishModule.run(network_name, &test_time).await; + }); join_all(vec![ handle_newaccount, handle_cointransfer, handle_nfttransfer, + handle_publishmodule, ]) .await; Ok(()) } -#[tokio::main] -async fn main() -> Result<()> { +fn main() -> Result<()> { + // create runtime + let runtime = Builder::new_multi_thread() + .worker_threads(*NUM_THREADS) + .enable_all() + .thread_stack_size(*STACK_SIZE) + .build()?; + // log metrics Logger::builder().level(Level::Info).build(); let _mp = MetricsPusher::start_for_local_run("api-tester"); - // test flows - let _ = test_flows(NetworkName::Testnet).await; - let _ = test_flows(NetworkName::Devnet).await; + // run tests + runtime.block_on(async { + let _ = test_flows(&runtime, NetworkName::Testnet).await; + let _ = test_flows(&runtime, NetworkName::Devnet).await; + }); Ok(()) } diff --git a/crates/aptos-api-tester/src/persistent_check.rs b/crates/aptos-api-tester/src/persistent_check.rs index 9e1873af2cc270..5eeae79f141ca8 100644 --- a/crates/aptos-api-tester/src/persistent_check.rs +++ b/crates/aptos-api-tester/src/persistent_check.rs @@ -1,16 +1,17 @@ // Copyright © Aptos Foundation -use crate::{fail_message::ERROR_COULD_NOT_CHECK, utils::TestFailure}; +use crate::{ + consts::{PERSISTENCY_TIMEOUT, SLEEP_PER_CYCLE}, + fail_message::ERROR_COULD_NOT_CHECK, + utils::TestFailure, +}; use anyhow::anyhow; use aptos_api_types::HexEncodedBytes; use aptos_rest_client::Client; use aptos_sdk::{token_client::TokenClient, types::LocalAccount}; use aptos_types::account_address::AccountAddress; use futures::Future; -use std::time::Duration; -use tokio::time::Instant; - -static PERSISTENCY_TIMEOUT: Duration = Duration::from_secs(30); +use tokio::time::{sleep, Instant}; pub async fn account<'a, 'b, F, Fut>( step: &str, @@ -27,11 +28,12 @@ where let timer = Instant::now(); // try to get a good result - while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT { + while Instant::now().duration_since(timer) < *PERSISTENCY_TIMEOUT { result = f(client, account).await; if result.is_ok() { break; } + sleep(*SLEEP_PER_CYCLE).await; } // return last failure if no good result occurs @@ -53,11 +55,40 @@ where let timer = Instant::now(); // try to get a good result - while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT { + while Instant::now().duration_since(timer) < *PERSISTENCY_TIMEOUT { result = f(client, address).await; if result.is_ok() { break; } + sleep(*SLEEP_PER_CYCLE).await; + } + + // return last failure if no good result occurs + result +} + +pub async fn address_address<'a, F, Fut>( + step: &str, + f: F, + client: &'a Client, + address: AccountAddress, + address2: AccountAddress, +) -> Result<(), TestFailure> +where + F: Fn(&'a Client, AccountAddress, AccountAddress) -> Fut, + Fut: Future>, +{ + // set a default error in case checks never start + let mut result: Result<(), TestFailure> = Err(could_not_check(step)); + let timer = Instant::now(); + + // try to get a good result + while Instant::now().duration_since(timer) < *PERSISTENCY_TIMEOUT { + result = f(client, address, address2).await; + if result.is_ok() { + break; + } + sleep(*SLEEP_PER_CYCLE).await; } // return last failure if no good result occurs @@ -80,11 +111,12 @@ where let timer = Instant::now(); // try to get a good result - while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT { + while Instant::now().duration_since(timer) < *PERSISTENCY_TIMEOUT { result = f(client, address, bytes).await; if result.is_ok() { break; } + sleep(*SLEEP_PER_CYCLE).await; } // return last failure if no good result occurs @@ -107,11 +139,12 @@ where let timer = Instant::now(); // try to get a good result - while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT { + while Instant::now().duration_since(timer) < *PERSISTENCY_TIMEOUT { result = f(client, address, version).await; if result.is_ok() { break; } + sleep(*SLEEP_PER_CYCLE).await; } // return last failure if no good result occurs @@ -133,11 +166,12 @@ where let timer = Instant::now(); // try to get a good result - while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT { + while Instant::now().duration_since(timer) < *PERSISTENCY_TIMEOUT { result = f(token_client, address).await; if result.is_ok() { break; } + sleep(*SLEEP_PER_CYCLE).await; } // return last failure if no good result occurs @@ -160,11 +194,12 @@ where let timer = Instant::now(); // try to get a good result - while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT { + while Instant::now().duration_since(timer) < *PERSISTENCY_TIMEOUT { result = f(token_client, address, address2).await; if result.is_ok() { break; } + sleep(*SLEEP_PER_CYCLE).await; } // return last failure if no good result occurs diff --git a/crates/aptos-api-tester/src/tests/coin_transfer.rs b/crates/aptos-api-tester/src/tests/coin_transfer.rs index 4a23fa331215b2..38903c5149a19d 100644 --- a/crates/aptos-api-tester/src/tests/coin_transfer.rs +++ b/crates/aptos-api-tester/src/tests/coin_transfer.rs @@ -1,15 +1,16 @@ // Copyright © Aptos Foundation use crate::{ + consts::FUND_AMOUNT, fail_message::{ ERROR_COULD_NOT_CREATE_ACCOUNT, ERROR_COULD_NOT_CREATE_TRANSACTION, ERROR_COULD_NOT_FINISH_TRANSACTION, ERROR_COULD_NOT_FUND_ACCOUNT, ERROR_NO_BALANCE, ERROR_NO_VERSION, FAIL_WRONG_BALANCE, FAIL_WRONG_BALANCE_AT_VERSION, }, - persistent_check, + persistent_check, time_fn, utils::{ - create_account, create_and_fund_account, get_client, get_faucet_client, NetworkName, - TestFailure, + check_balance, create_account, create_and_fund_account, emit_step_metrics, get_client, + get_faucet_client, NetworkName, TestFailure, TestName, }, }; use anyhow::{anyhow, Result}; @@ -24,32 +25,78 @@ static TRANSFER_AMOUNT: u64 = 1_000; /// Tests coin transfer. Checks that: /// - receiver balance reflects transferred amount /// - receiver balance shows correct amount at the previous version -pub async fn test(network_name: NetworkName) -> Result<(), TestFailure> { +pub async fn test(network_name: NetworkName, run_id: &str) -> Result<(), TestFailure> { // setup - let (client, mut account, receiver) = setup(network_name).await?; + let (client, mut account, receiver) = emit_step_metrics( + time_fn!(setup, network_name), + TestName::CoinTransfer, + "setup", + network_name, + run_id, + )?; let coin_client = CoinClient::new(&client); + // check account data persistently + emit_step_metrics( + time_fn!( + persistent_check::address_address, + "check_account_data", + check_account_data, + &client, + account.address(), + receiver + ), + TestName::CoinTransfer, + "check_account_data", + network_name, + run_id, + )?; + // transfer coins to the receiver - let version = transfer_coins(&client, &coin_client, &mut account, receiver).await?; + let version = emit_step_metrics( + time_fn!( + transfer_coins, + &client, + &coin_client, + &mut account, + receiver + ), + TestName::CoinTransfer, + "transfer_coins", + network_name, + run_id, + )?; // check receiver balance persistently - persistent_check::address( + emit_step_metrics( + time_fn!( + persistent_check::address, + "check_account_balance", + check_account_balance, + &client, + receiver + ), + TestName::CoinTransfer, "check_account_balance", - check_account_balance, - &client, - receiver, - ) - .await?; + network_name, + run_id, + )?; // check receiver balance at previous version persistently - persistent_check::address_version( + emit_step_metrics( + time_fn!( + persistent_check::address_version, + "check_account_balance_at_version", + check_account_balance_at_version, + &client, + receiver, + version + ), + TestName::CoinTransfer, "check_account_balance_at_version", - check_account_balance_at_version, - &client, - receiver, - version, - ) - .await?; + network_name, + run_id, + )?; Ok(()) } @@ -74,6 +121,10 @@ async fn setup( return Err(e.into()); }, }; + info!( + "test: coin_transfer part: setup creating account: {}", + account.address() + ); // create receiver let receiver = match create_account(&faucet_client).await { @@ -86,10 +137,25 @@ async fn setup( return Err(e.into()); }, }; + info!( + "test: coin_transfer part: setup creating receiver: {}", + receiver + ); Ok((client, account, receiver)) } +async fn check_account_data( + client: &Client, + account: AccountAddress, + receiver: AccountAddress, +) -> Result<(), TestFailure> { + check_balance(TestName::CoinTransfer, client, account, U64(FUND_AMOUNT)).await?; + check_balance(TestName::CoinTransfer, client, receiver, U64(0)).await?; + + Ok(()) +} + async fn transfer_coins( client: &Client, coin_client: &CoinClient<'_>, diff --git a/crates/aptos-api-tester/src/tests/new_account.rs b/crates/aptos-api-tester/src/tests/new_account.rs index 4c1a557f00b4dd..14b5e8b964b2ae 100644 --- a/crates/aptos-api-tester/src/tests/new_account.rs +++ b/crates/aptos-api-tester/src/tests/new_account.rs @@ -1,12 +1,16 @@ // Copyright © Aptos Foundation use crate::{ + consts::FUND_AMOUNT, fail_message::{ ERROR_COULD_NOT_CREATE_ACCOUNT, ERROR_COULD_NOT_FUND_ACCOUNT, ERROR_NO_ACCOUNT_DATA, - ERROR_NO_BALANCE, FAIL_WRONG_ACCOUNT_DATA, FAIL_WRONG_BALANCE, + FAIL_WRONG_ACCOUNT_DATA, + }, + persistent_check, time_fn, + utils::{ + check_balance, create_account, emit_step_metrics, get_client, get_faucet_client, + NetworkName, TestFailure, TestName, }, - persistent_check, - utils::{create_account, get_client, get_faucet_client, NetworkName, TestFailure}, }; use aptos_api_types::U64; use aptos_logger::info; @@ -14,29 +18,57 @@ use aptos_rest_client::{Account, Client, FaucetClient}; use aptos_sdk::types::LocalAccount; use aptos_types::account_address::AccountAddress; -static FUND_AMOUNT: u64 = 1_000_000; - /// Tests new account creation. Checks that: /// - account data exists /// - account balance reflects funded amount -pub async fn test(network_name: NetworkName) -> Result<(), TestFailure> { +pub async fn test(network_name: NetworkName, run_id: &str) -> Result<(), TestFailure> { // setup - let (client, faucet_client, account) = setup(network_name).await?; + let (client, faucet_client, account) = emit_step_metrics( + time_fn!(setup, network_name), + TestName::NewAccount, + "setup", + network_name, + run_id, + )?; // check account data persistently - persistent_check::account("check_account_data", check_account_data, &client, &account).await?; + emit_step_metrics( + time_fn!( + persistent_check::account, + "check_account_data", + check_account_data, + &client, + &account + ), + TestName::NewAccount, + "check_account_data", + network_name, + run_id, + )?; // fund account - fund(&faucet_client, account.address()).await?; + emit_step_metrics( + time_fn!(fund, &faucet_client, account.address()), + TestName::NewAccount, + "fund", + network_name, + run_id, + )?; // check account balance persistently - persistent_check::address( + emit_step_metrics( + time_fn!( + persistent_check::address, + "check_account_balance", + check_account_balance, + &client, + account.address() + ), + TestName::NewAccount, "check_account_balance", - check_account_balance, - &client, - account.address(), - ) - .await?; + network_name, + run_id, + )?; Ok(()) } @@ -61,6 +93,10 @@ async fn setup( return Err(e.into()); }, }; + info!( + "test: new_account part: setup creating account: {}", + account.address() + ); Ok((client, faucet_client, account)) } @@ -113,29 +149,5 @@ async fn check_account_balance( client: &Client, address: AccountAddress, ) -> Result<(), TestFailure> { - // expected - let expected = U64(FUND_AMOUNT); - - // actual - let actual = match client.get_account_balance(address).await { - Ok(response) => response.into_inner().coin.value, - Err(e) => { - info!( - "test: new_account part: check_account_balance ERROR: {}, with error {:?}", - ERROR_NO_BALANCE, e - ); - return Err(e.into()); - }, - }; - - // compare - if expected != actual { - info!( - "test: new_account part: check_account_balance FAIL: {}, expected {:?}, got {:?}", - FAIL_WRONG_BALANCE, expected, actual - ); - return Err(TestFailure::Fail(FAIL_WRONG_BALANCE)); - } - - Ok(()) + check_balance(TestName::NewAccount, client, address, U64(FUND_AMOUNT)).await } diff --git a/crates/aptos-api-tester/src/tests/nft_transfer.rs b/crates/aptos-api-tester/src/tests/nft_transfer.rs index 541966d703d42f..ebf79c9b12f743 100644 --- a/crates/aptos-api-tester/src/tests/nft_transfer.rs +++ b/crates/aptos-api-tester/src/tests/nft_transfer.rs @@ -1,14 +1,18 @@ // Copyright © Aptos Foundation use crate::{ + consts::FUND_AMOUNT, fail_message::{ ERROR_COULD_NOT_CREATE_TRANSACTION, ERROR_COULD_NOT_FINISH_TRANSACTION, ERROR_COULD_NOT_FUND_ACCOUNT, ERROR_NO_COLLECTION_DATA, ERROR_NO_TOKEN_BALANCE, ERROR_NO_TOKEN_DATA, FAIL_WRONG_COLLECTION_DATA, FAIL_WRONG_TOKEN_BALANCE, FAIL_WRONG_TOKEN_DATA, }, - persistent_check, - utils::{create_and_fund_account, get_client, get_faucet_client, NetworkName, TestFailure}, + persistent_check, time_fn, + utils::{ + check_balance, create_and_fund_account, emit_step_metrics, get_client, get_faucet_client, + NetworkName, TestFailure, TestName, + }, }; use aptos_api_types::U64; use aptos_logger::info; @@ -31,59 +35,141 @@ static OFFER_AMOUNT: u64 = 2; /// - collection data exists /// - token data exists /// - token balance reflects transferred amount -pub async fn test(network_name: NetworkName) -> Result<(), TestFailure> { +pub async fn test(network_name: NetworkName, run_id: &str) -> Result<(), TestFailure> { // setup - let (client, mut account, mut receiver) = setup(network_name).await?; + let (client, mut account, mut receiver) = emit_step_metrics( + time_fn!(setup, network_name), + TestName::NftTransfer, + "setup", + network_name, + run_id, + )?; let token_client = TokenClient::new(&client); + // check account data persistently + emit_step_metrics( + time_fn!( + persistent_check::address_address, + "check_account_data", + check_account_data, + &client, + account.address(), + receiver.address() + ), + TestName::NftTransfer, + "check_account_data", + network_name, + run_id, + )?; + // create collection - create_collection(&client, &token_client, &mut account).await?; + emit_step_metrics( + time_fn!(create_collection, &client, &token_client, &mut account), + TestName::NftTransfer, + "create_collection", + network_name, + run_id, + )?; // check collection metadata persistently - persistent_check::token_address( + emit_step_metrics( + time_fn!( + persistent_check::token_address, + "check_collection_metadata", + check_collection_metadata, + &token_client, + account.address() + ), + TestName::NftTransfer, "check_collection_metadata", - check_collection_metadata, - &token_client, - account.address(), - ) - .await?; + network_name, + run_id, + )?; // create token - create_token(&client, &token_client, &mut account).await?; + emit_step_metrics( + time_fn!(create_token, &client, &token_client, &mut account), + TestName::NftTransfer, + "create_token", + network_name, + run_id, + )?; // check token metadata persistently - persistent_check::token_address( + emit_step_metrics( + time_fn!( + persistent_check::token_address, + "check_token_address", + check_token_metadata, + &token_client, + account.address() + ), + TestName::NftTransfer, "check_token_metadata", - check_token_metadata, - &token_client, - account.address(), - ) - .await?; + network_name, + run_id, + )?; // offer token - offer_token(&client, &token_client, &mut account, receiver.address()).await?; + emit_step_metrics( + time_fn!( + offer_token, + &client, + &token_client, + &mut account, + receiver.address() + ), + TestName::NftTransfer, + "offer_token", + network_name, + run_id, + )?; // check senders balance persistently - persistent_check::token_address( + emit_step_metrics( + time_fn!( + persistent_check::token_address, + "check_sender_balance", + check_sender_balance, + &token_client, + account.address() + ), + TestName::NftTransfer, "check_sender_balance", - check_sender_balance, - &token_client, - account.address(), - ) - .await?; + network_name, + run_id, + )?; // claim token - claim_token(&client, &token_client, &mut receiver, account.address()).await?; + emit_step_metrics( + time_fn!( + claim_token, + &client, + &token_client, + &mut receiver, + account.address() + ), + TestName::NftTransfer, + "claim_token", + network_name, + run_id, + )?; // check receivers balance persistently - persistent_check::token_address_address( + emit_step_metrics( + time_fn!( + persistent_check::token_address_address, + "check_receiver_balance", + check_receiver_balance, + &token_client, + receiver.address(), + account.address() + ), + TestName::NftTransfer, "check_receiver_balance", - check_receiver_balance, - &token_client, - receiver.address(), - account.address(), - ) - .await?; + network_name, + run_id, + )?; Ok(()) } @@ -108,6 +194,10 @@ async fn setup( return Err(e.into()); }, }; + info!( + "test: nft_transfer part: setup creating account: {}", + account.address() + ); // create receiver let receiver = match create_and_fund_account(&faucet_client).await { @@ -120,10 +210,25 @@ async fn setup( return Err(e.into()); }, }; + info!( + "test: nft_transfer part: setup creating receiver: {}", + receiver.address() + ); Ok((client, account, receiver)) } +async fn check_account_data( + client: &Client, + account: AccountAddress, + receiver: AccountAddress, +) -> Result<(), TestFailure> { + check_balance(TestName::NftTransfer, client, account, U64(FUND_AMOUNT)).await?; + check_balance(TestName::NftTransfer, client, receiver, U64(FUND_AMOUNT)).await?; + + Ok(()) +} + async fn create_collection( client: &Client, token_client: &TokenClient<'_>, diff --git a/crates/aptos-api-tester/src/tests/publish_module.rs b/crates/aptos-api-tester/src/tests/publish_module.rs index 08e4653e357e76..e1214295518026 100644 --- a/crates/aptos-api-tester/src/tests/publish_module.rs +++ b/crates/aptos-api-tester/src/tests/publish_module.rs @@ -1,17 +1,21 @@ // Copyright © Aptos Foundation use crate::{ + consts::FUND_AMOUNT, fail_message::{ ERROR_COULD_NOT_BUILD_PACKAGE, ERROR_COULD_NOT_CREATE_TRANSACTION, ERROR_COULD_NOT_FINISH_TRANSACTION, ERROR_COULD_NOT_FUND_ACCOUNT, ERROR_COULD_NOT_SERIALIZE, ERROR_NO_BYTECODE, ERROR_NO_MESSAGE, ERROR_NO_METADATA, ERROR_NO_MODULE, FAIL_WRONG_MESSAGE, FAIL_WRONG_MODULE, }, - persistent_check, - utils::{create_and_fund_account, get_client, get_faucet_client, NetworkName, TestFailure}, + persistent_check, time_fn, + utils::{ + check_balance, create_and_fund_account, emit_step_metrics, get_client, get_faucet_client, + NetworkName, TestFailure, TestName, + }, }; use anyhow::{anyhow, Result}; -use aptos_api_types::HexEncodedBytes; +use aptos_api_types::{HexEncodedBytes, U64}; use aptos_cached_packages::aptos_stdlib::EntryFunctionCall; use aptos_framework::{BuildOptions, BuiltPackage}; use aptos_logger::info; @@ -31,31 +35,93 @@ use std::{collections::BTreeMap, path::PathBuf}; static MODULE_NAME: &str = "message"; static MESSAGE: &str = "test message"; -pub async fn test(network_name: NetworkName) -> Result<(), TestFailure> { +/// Tests nft transfer. Checks that: +/// - can publish module +/// - module data exists +/// - can interact with module +/// - interaction is reflected correctly +pub async fn test(network_name: NetworkName, run_id: &str) -> Result<(), TestFailure> { // setup - let (client, mut account) = setup(network_name).await?; + let (client, mut account) = emit_step_metrics( + time_fn!(setup, network_name), + TestName::PublishModule, + "setup", + network_name, + run_id, + )?; + + // check account data persistently + emit_step_metrics( + time_fn!( + persistent_check::address, + "check_account_data", + check_account_data, + &client, + account.address() + ), + TestName::PublishModule, + "check_account_data", + network_name, + run_id, + )?; // build module - let package = build_module(account.address()).await?; + let package = emit_step_metrics( + time_fn!(build_module, account.address()), + TestName::PublishModule, + "build_module", + network_name, + run_id, + )?; // publish module - let blob = publish_module(&client, &mut account, package).await?; + let blob = emit_step_metrics( + time_fn!(publish_module, &client, &mut account, package), + TestName::PublishModule, + "publish_module", + network_name, + run_id, + )?; // check module data persistently - persistent_check::address_bytes( + emit_step_metrics( + time_fn!( + persistent_check::address_bytes, + "check_module_data", + check_module_data, + &client, + account.address(), + &blob + ), + TestName::PublishModule, "check_module_data", - check_module_data, - &client, - account.address(), - &blob, - ) - .await?; + network_name, + run_id, + )?; // set message - set_message(&client, &mut account).await?; + emit_step_metrics( + time_fn!(set_message, &client, &mut account), + TestName::PublishModule, + "set_message", + network_name, + run_id, + )?; // check message persistently - persistent_check::address("check_message", check_message, &client, account.address()).await?; + emit_step_metrics( + time_fn!( + persistent_check::address, + "check_message", + check_message, + &client, + account.address() + ), + TestName::PublishModule, + "check_message", + network_name, + run_id, + )?; Ok(()) } @@ -78,10 +144,20 @@ async fn setup(network_name: NetworkName) -> Result<(Client, LocalAccount), Test return Err(e.into()); }, }; + info!( + "test: publish_module part: setup creating account: {}", + account.address() + ); Ok((client, account)) } +async fn check_account_data(client: &Client, account: AccountAddress) -> Result<(), TestFailure> { + check_balance(TestName::PublishModule, client, account, U64(FUND_AMOUNT)).await?; + + Ok(()) +} + async fn build_module(address: AccountAddress) -> Result { // get file to compile let move_dir = PathBuf::from("./aptos-move/move-examples/hello_blockchain"); diff --git a/crates/aptos-api-tester/src/utils.rs b/crates/aptos-api-tester/src/utils.rs index eb85d945780021..db465e2e0c9436 100644 --- a/crates/aptos-api-tester/src/utils.rs +++ b/crates/aptos-api-tester/src/utils.rs @@ -1,38 +1,23 @@ // Copyright © Aptos Foundation -use crate::counters::{test_error, test_fail, test_latency, test_success}; +use crate::{ + consts::{ + DEVNET_FAUCET_URL, DEVNET_NODE_URL, FUND_AMOUNT, TESTNET_FAUCET_URL, TESTNET_NODE_URL, + }, + counters::{test_error, test_fail, test_latency, test_step_latency, test_success}, + fail_message::{ERROR_NO_BALANCE, FAIL_WRONG_BALANCE}, + tests::{coin_transfer, new_account, nft_transfer, publish_module}, + time_fn, +}; use anyhow::Result; +use aptos_api_types::U64; +use aptos_logger::info; use aptos_rest_client::{error::RestError, Client, FaucetClient}; use aptos_sdk::types::LocalAccount; -use once_cell::sync::Lazy; +use aptos_types::account_address::AccountAddress; use std::env; -use url::Url; - -// network urls -pub static DEVNET_NODE_URL: Lazy = - Lazy::new(|| Url::parse("https://fullnode.devnet.aptoslabs.com").unwrap()); -pub static DEVNET_FAUCET_URL: Lazy = - Lazy::new(|| Url::parse("https://faucet.devnet.aptoslabs.com").unwrap()); -pub static TESTNET_NODE_URL: Lazy = - Lazy::new(|| Url::parse("https://fullnode.testnet.aptoslabs.com").unwrap()); -pub static TESTNET_FAUCET_URL: Lazy = - Lazy::new(|| Url::parse("https://faucet.testnet.aptoslabs.com").unwrap()); -#[derive(Debug)] -pub enum TestResult { - Success, - Fail(&'static str), - Error(anyhow::Error), -} - -impl From for TestResult { - fn from(f: TestFailure) -> TestResult { - match f { - TestFailure::Fail(f) => TestResult::Fail(f), - TestFailure::Error(e) => TestResult::Error(e), - } - } -} +// test failure #[derive(Debug)] pub enum TestFailure { @@ -52,6 +37,9 @@ impl From for TestFailure { } } +// test name + +#[derive(Clone, Copy)] pub enum TestName { NewAccount, CoinTransfer, @@ -59,6 +47,19 @@ pub enum TestName { PublishModule, } +impl TestName { + pub async fn run(&self, network_name: NetworkName, run_id: &str) { + let output = match &self { + TestName::NewAccount => time_fn!(new_account::test, network_name, run_id), + TestName::CoinTransfer => time_fn!(coin_transfer::test, network_name, run_id), + TestName::NftTransfer => time_fn!(nft_transfer::test, network_name, run_id), + TestName::PublishModule => time_fn!(publish_module::test, network_name, run_id), + }; + + emit_test_metrics(output, *self, network_name, run_id); + } +} + impl ToString for TestName { fn to_string(&self) -> String { match &self { @@ -70,6 +71,8 @@ impl ToString for TestName { } } +// network name + #[derive(Clone, Copy)] pub enum NetworkName { Testnet, @@ -85,37 +88,9 @@ impl ToString for NetworkName { } } -// Set metrics based on the result. -pub fn set_metrics( - output: &TestResult, - test_name: &str, - network_name: &str, - start_time: &str, - time: f64, -) { - match output { - TestResult::Success => { - test_success(test_name, network_name, start_time).observe(1_f64); - test_fail(test_name, network_name, start_time).observe(0_f64); - test_error(test_name, network_name, start_time).observe(0_f64); - test_latency(test_name, network_name, start_time, "success").observe(time); - }, - TestResult::Fail(_) => { - test_success(test_name, network_name, start_time).observe(0_f64); - test_fail(test_name, network_name, start_time).observe(1_f64); - test_error(test_name, network_name, start_time).observe(0_f64); - test_latency(test_name, network_name, start_time, "fail").observe(time); - }, - TestResult::Error(_) => { - test_success(test_name, network_name, start_time).observe(0_f64); - test_fail(test_name, network_name, start_time).observe(0_f64); - test_error(test_name, network_name, start_time).observe(1_f64); - test_latency(test_name, network_name, start_time, "error").observe(time); - }, - } -} +// setup helpers -// Create a REST client. +/// Create a REST client. pub fn get_client(network_name: NetworkName) -> Client { match network_name { NetworkName::Testnet => Client::new(TESTNET_NODE_URL.clone()), @@ -123,7 +98,7 @@ pub fn get_client(network_name: NetworkName) -> Client { } } -// Create a faucet client. +/// Create a faucet client. pub fn get_faucet_client(network_name: NetworkName) -> FaucetClient { match network_name { NetworkName::Testnet => { @@ -140,7 +115,7 @@ pub fn get_faucet_client(network_name: NetworkName) -> FaucetClient { } } -// Create an account with zero balance. +/// Create an account with zero balance. pub async fn create_account(faucet_client: &FaucetClient) -> Result { let account = LocalAccount::generate(&mut rand::rngs::OsRng); faucet_client.create_account(account.address()).await?; @@ -148,10 +123,147 @@ pub async fn create_account(faucet_client: &FaucetClient) -> Result Result { let account = LocalAccount::generate(&mut rand::rngs::OsRng); - faucet_client.fund(account.address(), 100_000_000).await?; + faucet_client.fund(account.address(), FUND_AMOUNT).await?; Ok(account) } + +/// Check account balance. +pub async fn check_balance( + test_name: TestName, + client: &Client, + address: AccountAddress, + expected: U64, +) -> Result<(), TestFailure> { + // actual + let actual = match client.get_account_balance(address).await { + Ok(response) => response.into_inner().coin.value, + Err(e) => { + info!( + "test: {} part: check_account_data ERROR: {}, with error {:?}", + &test_name.to_string(), + ERROR_NO_BALANCE, + e + ); + return Err(e.into()); + }, + }; + + // compare + if expected != actual { + info!( + "test: {} part: check_account_data FAIL: {}, expected {:?}, got {:?}", + &test_name.to_string(), + FAIL_WRONG_BALANCE, + expected, + actual + ); + return Err(TestFailure::Fail(FAIL_WRONG_BALANCE)); + } + + Ok(()) +} + +// metrics helpers + +/// Emit metrics based on test result. +pub fn emit_test_metrics( + output: (Result<(), TestFailure>, f64), + test_name: TestName, + network_name: NetworkName, + run_id: &str, +) { + // deconstruct + let (result, time) = output; + + // emit success rate and get result word + let result_label = match result { + Ok(_) => { + test_success(&test_name.to_string(), &network_name.to_string(), run_id).observe(1_f64); + test_fail(&test_name.to_string(), &network_name.to_string(), run_id).observe(0_f64); + test_error(&test_name.to_string(), &network_name.to_string(), run_id).observe(0_f64); + + "success" + }, + Err(e) => match e { + TestFailure::Fail(_) => { + test_success(&test_name.to_string(), &network_name.to_string(), run_id) + .observe(0_f64); + test_fail(&test_name.to_string(), &network_name.to_string(), run_id).observe(1_f64); + test_error(&test_name.to_string(), &network_name.to_string(), run_id) + .observe(0_f64); + + "fail" + }, + TestFailure::Error(_) => { + test_success(&test_name.to_string(), &network_name.to_string(), run_id) + .observe(0_f64); + test_fail(&test_name.to_string(), &network_name.to_string(), run_id).observe(0_f64); + test_error(&test_name.to_string(), &network_name.to_string(), run_id) + .observe(1_f64); + + "error" + }, + }, + }; + + // log result + info!( + "----- TEST FINISHED test: {} result: {} time: {} -----", + test_name.to_string(), + result_label, + time, + ); + + // emit latency + test_latency( + &test_name.to_string(), + &network_name.to_string(), + run_id, + result_label, + ) + .observe(time); +} + +/// Emit metrics based on result. +pub fn emit_step_metrics( + output: (Result, f64), + test_name: TestName, + step_name: &str, + network_name: NetworkName, + run_id: &str, +) -> Result { + // deconstruct and get result word + let (result, time) = output; + let result_label = match &result { + Ok(_) => "success", + Err(e) => match e { + TestFailure::Fail(_) => "fail", + TestFailure::Error(_) => "error", + }, + }; + + // log result + info!( + "STEP FINISHED test: {} step: {} result: {} time: {}", + test_name.to_string(), + step_name, + result_label, + time, + ); + + // emit latency + test_step_latency( + &test_name.to_string(), + step_name, + &network_name.to_string(), + run_id, + result_label, + ) + .observe(time); + + result +} diff --git a/crates/aptos-rest-client/src/lib.rs b/crates/aptos-rest-client/src/lib.rs index 1a6f653636f475..43ef9e5d5ce58c 100644 --- a/crates/aptos-rest-client/src/lib.rs +++ b/crates/aptos-rest-client/src/lib.rs @@ -566,7 +566,8 @@ impl Client { F: Fn(HashValue) -> Fut, Fut: Future>>, { - const DEFAULT_DELAY: Duration = Duration::from_millis(500); + // note: make this configurable + const DEFAULT_DELAY: Duration = Duration::from_millis(100); let mut reached_mempool = false; let start = std::time::Instant::now(); loop {