[api] Add support for strictness level for consistency on API tests (#…

…9444)

Description
This commit builds threads and run ID on top of the consistent API testing introduced here. See proposal for more high level information.

One problem we saw in the dashboard is that the problems we observed were regarding eventual consistency. We added support for adjusting the strictness level for such errors.

Changes:

Refactor of tests into tests module and decomposing into steps (setting up for next PR, individual step timing).
Persistent checks for information retrieval.
Added token support for Testnet faucet.

crates/aptos-api-tester/src/counters.rs

@@ -7,55 +7,50 @@ pub static API_TEST_SUCCESS: Lazy<HistogramVec> = Lazy::new(|| {
     register_histogram_vec!(
         "api_test_success",
         "Number of user flows which succesfully passed",
-        &["test_name", "network_name", "start_time"],
+        &["test_name", "network_name", "run_id"],
     )
     .unwrap()
 });
 
-pub fn test_success(test_name: &str, network_name: &str, start_time: &str) -> Histogram {
-    API_TEST_SUCCESS.with_label_values(&[test_name, network_name, start_time])
+pub fn test_success(test_name: &str, network_name: &str, run_id: &str) -> Histogram {
+    API_TEST_SUCCESS.with_label_values(&[test_name, network_name, run_id])
 }
 
 pub static API_TEST_FAIL: Lazy<HistogramVec> = Lazy::new(|| {
     register_histogram_vec!(
         "api_test_fail",
         "Number of user flows which failed checks",
-        &["test_name", "network_name", "start_time"],
+        &["test_name", "network_name", "run_id"],
     )
     .unwrap()
 });
 
-pub fn test_fail(test_name: &str, network_name: &str, start_time: &str) -> Histogram {
-    API_TEST_FAIL.with_label_values(&[test_name, network_name, start_time])
+pub fn test_fail(test_name: &str, network_name: &str, run_id: &str) -> Histogram {
+    API_TEST_FAIL.with_label_values(&[test_name, network_name, run_id])
 }
 
 pub static API_TEST_ERROR: Lazy<HistogramVec> = Lazy::new(|| {
     register_histogram_vec!("api_test_error", "Number of user flows which crashed", &[
         "test_name",
         "network_name",
-        "start_time"
+        "run_id"
     ],)
     .unwrap()
 });
 
-pub fn test_error(test_name: &str, network_name: &str, start_time: &str) -> Histogram {
-    API_TEST_ERROR.with_label_values(&[test_name, network_name, start_time])
+pub fn test_error(test_name: &str, network_name: &str, run_id: &str) -> Histogram {
+    API_TEST_ERROR.with_label_values(&[test_name, network_name, run_id])
 }
 
 pub static API_TEST_LATENCY: Lazy<HistogramVec> = Lazy::new(|| {
     register_histogram_vec!(
         "api_test_latency",
         "Time it takes to complete a user flow",
-        &["test_name", "network_name", "start_time", "result"],
+        &["test_name", "network_name", "run_id", "result"],
     )
     .unwrap()
 });
 
-pub fn test_latency(
-    test_name: &str,
-    network_name: &str,
-    start_time: &str,
-    result: &str,
-) -> Histogram {
-    API_TEST_LATENCY.with_label_values(&[test_name, network_name, start_time, result])
+pub fn test_latency(test_name: &str, network_name: &str, run_id: &str, result: &str) -> Histogram {
+    API_TEST_LATENCY.with_label_values(&[test_name, network_name, run_id, result])
 }

crates/aptos-api-tester/src/fail_message.rs

@@ -0,0 +1,27 @@
+// Copyright © Aptos Foundation
+
+pub static FAIL_WRONG_ACCOUNT_DATA: &str = "wrong account data";
+pub static FAIL_WRONG_BALANCE: &str = "wrong balance";
+pub static FAIL_WRONG_BALANCE_AT_VERSION: &str = "wrong balance at version";
+pub static FAIL_WRONG_COLLECTION_DATA: &str = "wrong collection data";
+pub static FAIL_WRONG_MESSAGE: &str = "wrong message";
+pub static FAIL_WRONG_MODULE: &str = "wrong module";
+pub static FAIL_WRONG_TOKEN_BALANCE: &str = "wrong token balance";
+pub static FAIL_WRONG_TOKEN_DATA: &str = "wrong token data";
+pub static ERROR_COULD_NOT_BUILD_PACKAGE: &str = "failed to build package";
+pub static ERROR_COULD_NOT_CHECK: &str = "persistency check never started";
+pub static ERROR_COULD_NOT_CREATE_ACCOUNT: &str = "failed to create account";
+pub static ERROR_COULD_NOT_CREATE_TRANSACTION: &str = "failed to create transaction";
+pub static ERROR_COULD_NOT_FINISH_TRANSACTION: &str = "failed to finish transaction";
+pub static ERROR_COULD_NOT_FUND_ACCOUNT: &str = "failed to fund account";
+pub static ERROR_COULD_NOT_SERIALIZE: &str = "failed to serialize";
+pub static ERROR_NO_ACCOUNT_DATA: &str = "can't find account data";
+pub static ERROR_NO_BALANCE: &str = "can't find account balance";
+pub static ERROR_NO_BYTECODE: &str = "can't find bytecode";
+pub static ERROR_NO_COLLECTION_DATA: &str = "can't find collection data";
+pub static ERROR_NO_MESSAGE: &str = "can't find message";
+pub static ERROR_NO_METADATA: &str = "can't find metadata";
+pub static ERROR_NO_MODULE: &str = "can't find module";
+pub static ERROR_NO_TOKEN_BALANCE: &str = "can't find token balance";
+pub static ERROR_NO_TOKEN_DATA: &str = "can't find token data";
+pub static ERROR_NO_VERSION: &str = "can't find transaction version";

crates/aptos-api-tester/src/main.rs

@@ -4,15 +4,13 @@
 #![forbid(unsafe_code)]
 
 mod counters;
+mod fail_message;
+mod persistent_check;
 mod tests;
-mod testsetups;
 mod utils;
 
 use crate::{
-    testsetups::{
-        setup_and_run_cointransfer, setup_and_run_newaccount, setup_and_run_nfttransfer,
-        setup_and_run_publishmodule,
-    },
+    tests::{coin_transfer, new_account, nft_transfer, publish_module},
     utils::{set_metrics, NetworkName, TestFailure, TestName, TestResult},
 };
 use anyhow::Result;
@@ -28,7 +26,7 @@ use std::{
 async fn process_result<Fut: Future<Output = Result<(), TestFailure>>>(
     test_name: TestName,
     network_name: NetworkName,
-    start_time: &str,
+    run_id: &str,
     fut: Fut,
 ) {
     // start timer
@@ -51,7 +49,7 @@ async fn process_result<Fut: Future<Output = Result<(), TestFailure>>>(
         &output,
         &test_name.to_string(),
         &network_name.to_string(),
-        start_time,
+        run_id,
         time,
     );
     info!(
@@ -64,55 +62,55 @@ async fn process_result<Fut: Future<Output = Result<(), TestFailure>>>(
 }
 
 async fn test_flows(network_name: NetworkName) -> Result<()> {
-    let start_time = SystemTime::now()
+    let run_id = SystemTime::now()
         .duration_since(UNIX_EPOCH)?
         .as_secs()
         .to_string();
-    info!("testing {} at {}", network_name.to_string(), start_time);
+    info!("testing {} at {}", network_name.to_string(), run_id);
 
     // Test new account creation and funding
-    let test_time = start_time.clone();
+    let test_time = run_id.clone();
     let handle_newaccount = tokio::spawn(async move {
         process_result(
             TestName::NewAccount,
             network_name,
             &test_time,
-            setup_and_run_newaccount(network_name),
+            new_account::test(network_name),
         )
         .await;
     });
 
     // Flow 1: Coin transfer
-    let test_time = start_time.clone();
+    let test_time = run_id.clone();
     let handle_cointransfer = tokio::spawn(async move {
         process_result(
             TestName::CoinTransfer,
             network_name,
             &test_time,
-            setup_and_run_cointransfer(network_name),
+            coin_transfer::test(network_name),
         )
         .await;
     });
 
     // Flow 2: NFT transfer
-    let test_time = start_time.clone();
+    let test_time = run_id.clone();
     let handle_nfttransfer = tokio::spawn(async move {
         process_result(
             TestName::NftTransfer,
             network_name,
             &test_time,
-            setup_and_run_nfttransfer(network_name),
+            nft_transfer::test(network_name),
         )
         .await;
     });
 
     // Flow 3: Publishing module
-    let test_time = start_time.clone();
+    let test_time = run_id.clone();
     process_result(
         TestName::PublishModule,
         network_name,
         &test_time,
-        setup_and_run_publishmodule(network_name),
+        publish_module::test(network_name),
     )
     .await;
 

crates/aptos-api-tester/src/persistent_check.rs

@@ -0,0 +1,178 @@
+// Copyright © Aptos Foundation
+
+use crate::{fail_message::ERROR_COULD_NOT_CHECK, utils::TestFailure};
+use anyhow::anyhow;
+use aptos_api_types::HexEncodedBytes;
+use aptos_rest_client::Client;
+use aptos_sdk::{token_client::TokenClient, types::LocalAccount};
+use aptos_types::account_address::AccountAddress;
+use futures::Future;
+use std::time::Duration;
+use tokio::time::Instant;
+
+static PERSISTENCY_TIMEOUT: Duration = Duration::from_secs(30);
+
+pub async fn account<'a, 'b, F, Fut>(
+    step: &str,
+    f: F,
+    client: &'a Client,
+    account: &'b LocalAccount,
+) -> Result<(), TestFailure>
+where
+    F: Fn(&'a Client, &'b LocalAccount) -> Fut,
+    Fut: Future<Output = Result<(), TestFailure>>,
+{
+    // set a default error in case checks never start
+    let mut result: Result<(), TestFailure> = Err(could_not_check(step));
+    let timer = Instant::now();
+
+    // try to get a good result
+    while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT {
+        result = f(client, account).await;
+        if result.is_ok() {
+            break;
+        }
+    }
+
+    // return last failure if no good result occurs
+    result
+}
+
+pub async fn address<'a, F, Fut>(
+    step: &str,
+    f: F,
+    client: &'a Client,
+    address: AccountAddress,
+) -> Result<(), TestFailure>
+where
+    F: Fn(&'a Client, AccountAddress) -> Fut,
+    Fut: Future<Output = Result<(), TestFailure>>,
+{
+    // set a default error in case checks never start
+    let mut result: Result<(), TestFailure> = Err(could_not_check(step));
+    let timer = Instant::now();
+
+    // try to get a good result
+    while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT {
+        result = f(client, address).await;
+        if result.is_ok() {
+            break;
+        }
+    }
+
+    // return last failure if no good result occurs
+    result
+}
+
+pub async fn address_bytes<'a, 'b, F, Fut>(
+    step: &str,
+    f: F,
+    client: &'a Client,
+    address: AccountAddress,
+    bytes: &'b HexEncodedBytes,
+) -> Result<(), TestFailure>
+where
+    F: Fn(&'a Client, AccountAddress, &'b HexEncodedBytes) -> Fut,
+    Fut: Future<Output = Result<(), TestFailure>>,
+{
+    // set a default error in case checks never start
+    let mut result: Result<(), TestFailure> = Err(could_not_check(step));
+    let timer = Instant::now();
+
+    // try to get a good result
+    while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT {
+        result = f(client, address, bytes).await;
+        if result.is_ok() {
+            break;
+        }
+    }
+
+    // return last failure if no good result occurs
+    result
+}
+
+pub async fn address_version<'a, F, Fut>(
+    step: &str,
+    f: F,
+    client: &'a Client,
+    address: AccountAddress,
+    version: u64,
+) -> Result<(), TestFailure>
+where
+    F: Fn(&'a Client, AccountAddress, u64) -> Fut,
+    Fut: Future<Output = Result<(), TestFailure>>,
+{
+    // set a default error in case checks never start
+    let mut result: Result<(), TestFailure> = Err(could_not_check(step));
+    let timer = Instant::now();
+
+    // try to get a good result
+    while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT {
+        result = f(client, address, version).await;
+        if result.is_ok() {
+            break;
+        }
+    }
+
+    // return last failure if no good result occurs
+    result
+}
+
+pub async fn token_address<'a, F, Fut>(
+    step: &str,
+    f: F,
+    token_client: &'a TokenClient<'a>,
+    address: AccountAddress,
+) -> Result<(), TestFailure>
+where
+    F: Fn(&'a TokenClient<'a>, AccountAddress) -> Fut,
+    Fut: Future<Output = Result<(), TestFailure>>,
+{
+    // set a default error in case checks never start
+    let mut result: Result<(), TestFailure> = Err(could_not_check(step));
+    let timer = Instant::now();
+
+    // try to get a good result
+    while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT {
+        result = f(token_client, address).await;
+        if result.is_ok() {
+            break;
+        }
+    }
+
+    // return last failure if no good result occurs
+    result
+}
+
+pub async fn token_address_address<'a, F, Fut>(
+    step: &str,
+    f: F,
+    token_client: &'a TokenClient<'a>,
+    address: AccountAddress,
+    address2: AccountAddress,
+) -> Result<(), TestFailure>
+where
+    F: Fn(&'a TokenClient<'a>, AccountAddress, AccountAddress) -> Fut,
+    Fut: Future<Output = Result<(), TestFailure>>,
+{
+    // set a default error in case checks never start
+    let mut result: Result<(), TestFailure> = Err(could_not_check(step));
+    let timer = Instant::now();
+
+    // try to get a good result
+    while Instant::now().duration_since(timer) < PERSISTENCY_TIMEOUT {
+        result = f(token_client, address, address2).await;
+        if result.is_ok() {
+            break;
+        }
+    }
+
+    // return last failure if no good result occurs
+    result
+}
+
+// Utils
+
+fn could_not_check(step: &str) -> TestFailure {
+    anyhow!(format!("{} in step: {}", ERROR_COULD_NOT_CHECK, step)).into()
+}