From 151832bf4fac4facbffd978fd0ba48f0b4effe84 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Thu, 21 Nov 2024 22:40:19 -0800 Subject: [PATCH] Add a workload that randomly generates different transaction shape --- .../sui-benchmark/src/drivers/bench_driver.rs | 23 +- crates/sui-benchmark/src/options.rs | 3 + .../src/workloads/adversarial.rs | 2 +- .../src/workloads/batch_payment.rs | 2 +- .../sui-benchmark/src/workloads/delegation.rs | 2 +- .../src/workloads/expected_failure.rs | 3 +- crates/sui-benchmark/src/workloads/mod.rs | 1 + .../src/workloads/randomized_transaction.rs | 531 ++++++++++++++++++ .../sui-benchmark/src/workloads/randomness.rs | 2 +- .../src/workloads/shared_counter.rs | 2 +- .../src/workloads/shared_object_deletion.rs | 2 +- .../src/workloads/transfer_object.rs | 2 +- .../sui-benchmark/src/workloads/workload.rs | 3 +- .../src/workloads/workload_configuration.rs | 25 +- crates/sui-benchmark/tests/simtest.rs | 9 +- 15 files changed, 593 insertions(+), 19 deletions(-) create mode 100644 crates/sui-benchmark/src/workloads/randomized_transaction.rs diff --git a/crates/sui-benchmark/src/drivers/bench_driver.rs b/crates/sui-benchmark/src/drivers/bench_driver.rs index 148d79f51ae6a..1ab4de38c26ed 100644 --- a/crates/sui-benchmark/src/drivers/bench_driver.rs +++ b/crates/sui-benchmark/src/drivers/bench_driver.rs @@ -26,6 +26,7 @@ use crate::drivers::driver::Driver; use crate::drivers::HistogramWrapper; use crate::system_state_observer::SystemStateObserver; use crate::workloads::payload::Payload; +use crate::workloads::workload::ExpectedFailureType; use crate::workloads::{GroupID, WorkloadInfo}; use crate::{ExecutionEffects, ValidatorProxy}; use std::collections::{BTreeMap, VecDeque}; @@ -737,7 +738,10 @@ async fn run_bench_worker( -> NextOp { match result { Ok(effects) => { - assert!(payload.get_failure_type().is_none()); + assert!( + payload.get_failure_type().is_none() + || payload.get_failure_type() == Some(ExpectedFailureType::NoFailure) + ); let latency = start.elapsed(); let time_from_start = total_benchmark_start_time.elapsed(); @@ -796,8 +800,15 @@ async fn run_bench_worker( } } Err(err) => { - error!("{}", err); + tracing::error!( + "Transaction execution got error: {}. Transaction digest: {:?}", + err, + transaction.digest() + ); match payload.get_failure_type() { + Some(ExpectedFailureType::NoFailure) => { + panic!("Transaction failed unexpectedly"); + } Some(_) => { metrics_cloned .num_expected_error @@ -917,10 +928,10 @@ async fn run_bench_worker( if let Some(b) = retry_queue.pop_front() { let tx = b.0; let payload = b.1; - if payload.get_failure_type().is_some() { - num_expected_error_txes += 1; - } else { - num_error_txes += 1; + match payload.get_failure_type() { + Some(ExpectedFailureType::NoFailure) => num_error_txes += 1, + Some(_) => num_expected_error_txes += 1, + None => num_error_txes += 1, } num_submitted += 1; metrics_cloned.num_submitted.with_label_values(&[&payload.to_string()]).inc(); diff --git a/crates/sui-benchmark/src/options.rs b/crates/sui-benchmark/src/options.rs index 4bab34fd41930..1373d506ec351 100644 --- a/crates/sui-benchmark/src/options.rs +++ b/crates/sui-benchmark/src/options.rs @@ -184,6 +184,9 @@ pub enum RunSpec { // relative weight of expected failure transactions in the benchmark workload #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = [0])] expected_failure: Vec, + // relative weight of randomized transaction in the benchmark workload + #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = [0])] + randomized_transaction: Vec, // --- workload-specific options --- (TODO: use subcommands or similar) // 100 for max hotness i.e all requests target diff --git a/crates/sui-benchmark/src/workloads/adversarial.rs b/crates/sui-benchmark/src/workloads/adversarial.rs index 39746b89c31d5..5471ee60f5562 100644 --- a/crates/sui-benchmark/src/workloads/adversarial.rs +++ b/crates/sui-benchmark/src/workloads/adversarial.rs @@ -412,7 +412,7 @@ impl AdversarialWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/batch_payment.rs b/crates/sui-benchmark/src/workloads/batch_payment.rs index 94e58da2ae484..c4e068b99f15a 100644 --- a/crates/sui-benchmark/src/workloads/batch_payment.rs +++ b/crates/sui-benchmark/src/workloads/batch_payment.rs @@ -138,7 +138,7 @@ impl BatchPaymentWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/delegation.rs b/crates/sui-benchmark/src/workloads/delegation.rs index 24f21a120f12c..3e0fdd651e8b2 100644 --- a/crates/sui-benchmark/src/workloads/delegation.rs +++ b/crates/sui-benchmark/src/workloads/delegation.rs @@ -100,7 +100,7 @@ impl DelegationWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/expected_failure.rs b/crates/sui-benchmark/src/workloads/expected_failure.rs index ae200ccbe759b..f5354927a330b 100644 --- a/crates/sui-benchmark/src/workloads/expected_failure.rs +++ b/crates/sui-benchmark/src/workloads/expected_failure.rs @@ -54,6 +54,7 @@ impl ExpectedFailurePayload { tx } ExpectedFailureType::Random => unreachable!(), + ExpectedFailureType::NoFailure => unreachable!(), } } } @@ -112,7 +113,7 @@ impl ExpectedFailureWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/mod.rs b/crates/sui-benchmark/src/workloads/mod.rs index 368f2c96fe0c4..466cad016c606 100644 --- a/crates/sui-benchmark/src/workloads/mod.rs +++ b/crates/sui-benchmark/src/workloads/mod.rs @@ -6,6 +6,7 @@ pub mod batch_payment; pub mod delegation; pub mod expected_failure; pub mod payload; +pub mod randomized_transaction; pub mod randomness; pub mod shared_counter; pub mod shared_object_deletion; diff --git a/crates/sui-benchmark/src/workloads/randomized_transaction.rs b/crates/sui-benchmark/src/workloads/randomized_transaction.rs new file mode 100644 index 0000000000000..19f18cccdfd4e --- /dev/null +++ b/crates/sui-benchmark/src/workloads/randomized_transaction.rs @@ -0,0 +1,531 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::drivers::Interval; +use crate::system_state_observer::SystemStateObserver; +use crate::util::publish_basics_package; +use crate::workloads::payload::Payload; +use crate::workloads::workload::{ + ExpectedFailureType, Workload, WorkloadBuilder, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING, +}; +use crate::workloads::{Gas, GasCoinConfig, WorkloadBuilderInfo, WorkloadParams}; +use crate::{ExecutionEffects, ValidatorProxy}; +use async_trait::async_trait; +use futures::future::join_all; +use rand::Rng; +use std::sync::Arc; +use sui_test_transaction_builder::TestTransactionBuilder; +use sui_types::base_types::{ObjectID, ObjectRef, SequenceNumber, SuiAddress}; +use sui_types::crypto::{get_key_pair, AccountKeyPair}; +use sui_types::object::Owner; +use sui_types::programmable_transaction_builder::ProgrammableTransactionBuilder; +use sui_types::transaction::{CallArg, ObjectArg, Transaction}; +use sui_types::{Identifier, SUI_RANDOMNESS_STATE_OBJECT_ID}; +use tracing::{error, info}; + +use super::STORAGE_COST_PER_COUNTER; + +pub const MAX_GAS_IN_UNIT: u64 = 1_000_000_000; + +/// A workload that generates random transactions to test the system under different transaction patterns. +/// The workload can: +/// - Create shared counter objects +/// - Make random move calls to increment/read/delete counters +/// - Make calls to the randomness module +/// - Make native transfers +/// - Mix different numbers of pure and shared object inputs +/// - Generate multiple move calls per transaction +/// +/// The exact mix of operations is controlled by random selection within configured bounds. +/// +/// Different from adversarial workload, this workload is not designed to test the system under +/// malicious behaviors, but to test the system under different transaction patterns. +#[derive(Debug)] +pub struct RandomizedTransactionPayload { + package_id: ObjectID, + shared_objects: Vec, + owned_object: ObjectRef, + randomness_initial_shared_version: SequenceNumber, + transfer_to: SuiAddress, + gas: Gas, + system_state_observer: Arc, +} + +impl std::fmt::Display for RandomizedTransactionPayload { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "randomized_transaction") + } +} + +/// Config for a randomized transaction +#[derive(Debug)] +struct RandomizedTransactionConfig { + // Whether the transaction contains the owned object + contain_owned_object: bool, + // Number of pure inputs + num_pure_input: u64, + // Number of shared inputs + num_shared_inputs: u64, + // Number of move calls + num_move_calls: u64, +} + +fn generate_random_transaction_config( + num_shared_objects_exist: u64, +) -> RandomizedTransactionConfig { + let num_shared_inputs = rand::thread_rng().gen_range(0..=num_shared_objects_exist); + RandomizedTransactionConfig { + contain_owned_object: rand::thread_rng().gen_bool(0.5), + num_pure_input: rand::thread_rng().gen_range(0..=5), + num_shared_inputs, + num_move_calls: std::cmp::min(rand::thread_rng().gen_range(0..=3), num_shared_inputs), + } +} + +/// Type of move call to generate. +enum MoveCallType { + ContractCall, + Randomness, + NativeCall, +} + +/// Choose a random move call type. +fn choose_move_call_type(next_shared_input_index: usize, num_shared_inputs: u64) -> MoveCallType { + if next_shared_input_index < num_shared_inputs as usize { + match rand::thread_rng().gen_range(0..=2) { + 0 => MoveCallType::ContractCall, + 1 => MoveCallType::Randomness, + _ => MoveCallType::NativeCall, + } + } else { + match rand::thread_rng().gen_range(0..=1) { + 0 => MoveCallType::Randomness, + _ => MoveCallType::NativeCall, + } + } +} + +impl RandomizedTransactionPayload { + fn make_counter_move_call( + &mut self, + builder: &mut ProgrammableTransactionBuilder, + next_shared_input_index: usize, + ) { + // 33% chance to increment, 33% chance to set value, 33% chance to read value. + match rand::thread_rng().gen_range(0..=2) { + 0 => { + builder + .move_call( + self.package_id, + Identifier::new("counter").unwrap(), + Identifier::new("increment").unwrap(), + vec![], + vec![CallArg::Object(ObjectArg::SharedObject { + id: self.shared_objects[next_shared_input_index].0, + initial_shared_version: self.shared_objects[next_shared_input_index].1, + mutable: true, + })], + ) + .unwrap(); + } + 1 => { + builder + .move_call( + self.package_id, + Identifier::new("counter").unwrap(), + Identifier::new("set_value").unwrap(), + vec![], + vec![ + CallArg::Object(ObjectArg::SharedObject { + id: self.shared_objects[next_shared_input_index].0, + initial_shared_version: self.shared_objects + [next_shared_input_index] + .1, + mutable: true, + }), + CallArg::Pure((10_u64).to_le_bytes().to_vec()), + ], + ) + .unwrap(); + } + _ => { + builder + .move_call( + self.package_id, + Identifier::new("counter").unwrap(), + Identifier::new("value").unwrap(), + vec![], + vec![CallArg::Object(ObjectArg::SharedObject { + id: self.shared_objects[next_shared_input_index].0, + initial_shared_version: self.shared_objects[next_shared_input_index].1, + mutable: false, + })], + ) + .unwrap(); + } + } + } + + fn make_randomness_move_call(&mut self, builder: &mut ProgrammableTransactionBuilder) { + builder + .move_call( + self.package_id, + Identifier::new("random").unwrap(), + Identifier::new("new").unwrap(), + vec![], + vec![CallArg::Object(ObjectArg::SharedObject { + id: SUI_RANDOMNESS_STATE_OBJECT_ID, + initial_shared_version: self.randomness_initial_shared_version, + mutable: false, + })], + ) + .unwrap(); + } + + fn make_native_move_call(&mut self, builder: &mut ProgrammableTransactionBuilder) { + builder + .pay_sui( + vec![self.transfer_to], + vec![rand::thread_rng().gen_range(0..=1)], + ) + .unwrap(); + } +} + +impl Payload for RandomizedTransactionPayload { + fn make_new_payload(&mut self, effects: &ExecutionEffects) { + if !effects.is_ok() { + effects.print_gas_summary(); + error!( + "Randomized transaction failed... Status: {:?}", + effects.status() + ); + } + self.gas.0 = effects.gas_object().0; + + // Update owned object if it's mutated in this transaction + if let Some(owned_in_effects) = effects + .mutated() + .iter() + .find(|(object_ref, _)| object_ref.0 == self.owned_object.0) + .map(|x| x.0) + { + tracing::debug!("Owned object mutated: {:?}", owned_in_effects); + self.owned_object = owned_in_effects; + } + } + + fn make_transaction(&mut self) -> Transaction { + let rgp = self + .system_state_observer + .state + .borrow() + .reference_gas_price; + + let config = generate_random_transaction_config(self.shared_objects.len() as u64); + + let mut builder = ProgrammableTransactionBuilder::new(); + + // Generate inputs in addition to move calls. + if config.contain_owned_object { + builder + .obj(ObjectArg::ImmOrOwnedObject(self.owned_object)) + .unwrap(); + } + for i in 0..config.num_shared_inputs { + builder + .obj(ObjectArg::SharedObject { + id: self.shared_objects[i as usize].0, + initial_shared_version: self.shared_objects[i as usize].1, + mutable: rand::thread_rng().gen_bool(0.5), + }) + .unwrap(); + } + for _i in 0..config.num_pure_input { + let len = rand::thread_rng().gen_range(0..=3); + let mut bytes = vec![0u8; len]; + rand::thread_rng().fill(&mut bytes[..]); + builder.pure_bytes(bytes, false); + } + + // Generate move calls. + let mut next_shared_input_index: usize = 0; + for _i in 0..config.num_move_calls { + match choose_move_call_type(next_shared_input_index, config.num_shared_inputs) { + MoveCallType::ContractCall => { + self.make_counter_move_call(&mut builder, next_shared_input_index); + next_shared_input_index += 1; + } + MoveCallType::Randomness => { + self.make_randomness_move_call(&mut builder); + // TODO: add TransferObject move call after randomness command. + break; + } + MoveCallType::NativeCall => { + self.make_native_move_call(&mut builder); + } + } + } + let tx = builder.finish(); + + tracing::info!("Randomized transaction: {:?}", tx); + + let signed_tx = TestTransactionBuilder::new(self.gas.1, self.gas.0, rgp) + .programmable(tx) + .build_and_sign(self.gas.2.as_ref()); + + tracing::debug!("Signed transaction digest: {:?}", signed_tx.digest()); + signed_tx + } + + fn get_failure_type(&self) -> Option { + // We do not expect randomized transaction to fail + Some(ExpectedFailureType::NoFailure) + } +} + +#[derive(Debug)] +pub struct RandomizedTransactionWorkloadBuilder { + num_payloads: u64, + rgp: u64, +} + +impl RandomizedTransactionWorkloadBuilder { + pub fn from( + workload_weight: f32, + target_qps: u64, + num_workers: u64, + in_flight_ratio: u64, + reference_gas_price: u64, + duration: Interval, + group: u32, + ) -> Option { + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; + let num_workers = (workload_weight * num_workers as f32).ceil() as u64; + let max_ops = target_qps * in_flight_ratio; + + if max_ops == 0 || num_workers == 0 { + None + } else { + let workload_params = WorkloadParams { + group, + target_qps, + num_workers, + max_ops, + duration, + }; + let workload_builder = Box::>::from(Box::new( + RandomizedTransactionWorkloadBuilder { + num_payloads: max_ops, + rgp: reference_gas_price, + }, + )); + Some(WorkloadBuilderInfo { + workload_params, + workload_builder, + }) + } + } +} + +#[async_trait] +impl WorkloadBuilder for RandomizedTransactionWorkloadBuilder { + async fn generate_coin_config_for_init(&self) -> Vec { + let mut configs = vec![]; + + // Gas coin for publishing package + let (address, keypair) = get_key_pair(); + configs.push(GasCoinConfig { + amount: MAX_GAS_FOR_TESTING, + address, + keypair: Arc::new(keypair), + }); + + // Gas coins for creating counters + for _i in 0..self.num_payloads { + let (address, keypair) = get_key_pair(); + configs.push(GasCoinConfig { + amount: MAX_GAS_FOR_TESTING, + address, + keypair: Arc::new(keypair), + }); + } + configs + } + + async fn generate_coin_config_for_payloads(&self) -> Vec { + let mut configs = vec![]; + let amount = MAX_GAS_IN_UNIT * (self.rgp) + + ESTIMATED_COMPUTATION_COST + + STORAGE_COST_PER_COUNTER * self.num_payloads + + MAX_GAS_FOR_TESTING; + // Gas coins for running workload + for _i in 0..self.num_payloads { + let (address, keypair) = get_key_pair(); + configs.push(GasCoinConfig { + amount, + address, + keypair: Arc::new(keypair), + }); + } + configs + } + + async fn build( + &self, + init_gas: Vec, + payload_gas: Vec, + ) -> Box> { + Box::>::from(Box::new(RandomizedTransactionWorkload { + basics_package_id: None, + shared_objects: vec![], + owned_objects: vec![], + transfer_to: None, + init_gas, + payload_gas, + randomness_initial_shared_version: None, + })) + } +} + +#[derive(Debug)] +pub struct RandomizedTransactionWorkload { + pub basics_package_id: Option, + pub shared_objects: Vec, + pub owned_objects: Vec, + pub transfer_to: Option, + pub init_gas: Vec, + pub payload_gas: Vec, + pub randomness_initial_shared_version: Option, +} + +#[async_trait] +impl Workload for RandomizedTransactionWorkload { + async fn init( + &mut self, + proxy: Arc, + system_state_observer: Arc, + ) { + if self.basics_package_id.is_some() { + return; + } + let gas_price = system_state_observer.state.borrow().reference_gas_price; + let (head, tail) = self + .init_gas + .split_first() + .expect("Not enough gas to initialize randomized transaction workload"); + + // Publish basics package + info!("Publishing basics package"); + self.basics_package_id = Some( + publish_basics_package(head.0, proxy.clone(), head.1, &head.2, gas_price) + .await + .0, + ); + + // Create a transfer address + self.transfer_to = Some(get_key_pair::().0); + + // Create shared objects + { + let mut futures = vec![]; + for (gas, sender, keypair) in tail.iter() { + let transaction = TestTransactionBuilder::new(*sender, *gas, gas_price) + .call_counter_create(self.basics_package_id.unwrap()) + .build_and_sign(keypair.as_ref()); + let proxy_ref = proxy.clone(); + futures.push(async move { + proxy_ref + .execute_transaction_block(transaction) + .await + .unwrap() + .created()[0] + .0 + }); + } + self.shared_objects = join_all(futures).await; + } + + // create owned objects + { + let mut futures = vec![]; + for (gas, sender, keypair) in self.payload_gas.iter() { + let transaction = TestTransactionBuilder::new(*sender, *gas, gas_price) + .move_call( + self.basics_package_id.unwrap(), + "object_basics", + "create", + vec![ + CallArg::Pure(bcs::to_bytes(&(16_u64)).unwrap()), + CallArg::Pure(bcs::to_bytes(&sender).unwrap()), + ], + ) + .build_and_sign(keypair.as_ref()); + let proxy_ref = proxy.clone(); + futures.push(async move { + let execution_result = proxy_ref + .execute_transaction_block(transaction) + .await + .unwrap(); + let created_owned = execution_result.created()[0].0; + let updated_gas = execution_result.gas_object().0; + (created_owned, updated_gas) + }); + } + let results = join_all(futures).await; + self.owned_objects = results.iter().map(|x| x.0).collect(); + + // Update gas object in payload gas + for (payload_gas, result) in self.payload_gas.iter_mut().zip(results.iter()) { + payload_gas.0 = result.1; + } + } + + // Get randomness shared object initial version + if self.randomness_initial_shared_version.is_none() { + let obj = proxy + .get_object(SUI_RANDOMNESS_STATE_OBJECT_ID) + .await + .expect("Failed to get randomness object"); + let Owner::Shared { + initial_shared_version, + } = obj.owner() + else { + panic!("randomness object must be shared"); + }; + self.randomness_initial_shared_version = Some(*initial_shared_version); + } + + info!( + "Basics package id {:?}. Total shared objects created {:?}", + self.basics_package_id, + self.shared_objects.len() + ); + } + + async fn make_test_payloads( + &self, + _proxy: Arc, + system_state_observer: Arc, + ) -> Vec> { + info!("Creating randomized transaction payloads..."); + let mut payloads = vec![]; + + for (i, g) in self.payload_gas.iter().enumerate() { + payloads.push(Box::new(RandomizedTransactionPayload { + package_id: self.basics_package_id.unwrap(), + shared_objects: self.shared_objects.clone(), + owned_object: self.owned_objects[i], + randomness_initial_shared_version: self.randomness_initial_shared_version.unwrap(), + transfer_to: self.transfer_to.unwrap(), + gas: g.clone(), + system_state_observer: system_state_observer.clone(), + })); + } + + payloads + .into_iter() + .map(|b| Box::::from(b)) + .collect() + } +} diff --git a/crates/sui-benchmark/src/workloads/randomness.rs b/crates/sui-benchmark/src/workloads/randomness.rs index 6a812b7b4d636..3294fcfb28d2c 100644 --- a/crates/sui-benchmark/src/workloads/randomness.rs +++ b/crates/sui-benchmark/src/workloads/randomness.rs @@ -79,7 +79,7 @@ impl RandomnessWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/shared_counter.rs b/crates/sui-benchmark/src/workloads/shared_counter.rs index 5356d53b7184d..38f75fd9707d7 100644 --- a/crates/sui-benchmark/src/workloads/shared_counter.rs +++ b/crates/sui-benchmark/src/workloads/shared_counter.rs @@ -98,7 +98,7 @@ impl SharedCounterWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; let shared_counter_ratio = diff --git a/crates/sui-benchmark/src/workloads/shared_object_deletion.rs b/crates/sui-benchmark/src/workloads/shared_object_deletion.rs index 552cd86c85fa8..6db465fa62340 100644 --- a/crates/sui-benchmark/src/workloads/shared_object_deletion.rs +++ b/crates/sui-benchmark/src/workloads/shared_object_deletion.rs @@ -143,7 +143,7 @@ impl SharedCounterDeletionWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; let shared_counter_ratio = diff --git a/crates/sui-benchmark/src/workloads/transfer_object.rs b/crates/sui-benchmark/src/workloads/transfer_object.rs index 1835a5d0f17af..d02247154bcdf 100644 --- a/crates/sui-benchmark/src/workloads/transfer_object.rs +++ b/crates/sui-benchmark/src/workloads/transfer_object.rs @@ -108,7 +108,7 @@ impl TransferObjectWorkloadBuilder { duration: Interval, group: u32, ) -> Option { - let target_qps = (workload_weight * target_qps as f32) as u64; + let target_qps = (workload_weight * target_qps as f32).ceil() as u64; let num_workers = (workload_weight * num_workers as f32).ceil() as u64; let max_ops = target_qps * in_flight_ratio; if max_ops == 0 || num_workers == 0 { diff --git a/crates/sui-benchmark/src/workloads/workload.rs b/crates/sui-benchmark/src/workloads/workload.rs index d4c4cb507c270..35f475b2a1ad5 100644 --- a/crates/sui-benchmark/src/workloads/workload.rs +++ b/crates/sui-benchmark/src/workloads/workload.rs @@ -29,10 +29,11 @@ pub const STORAGE_COST_PER_COUNTER: u64 = 341 * 76 * 100; /// Used to estimate the budget required for each transaction. pub const ESTIMATED_COMPUTATION_COST: u64 = 1_000_000; -#[derive(Debug, EnumCountMacro, EnumIter, Clone, Copy)] +#[derive(Debug, EnumCountMacro, EnumIter, Clone, Copy, PartialEq)] pub enum ExpectedFailureType { Random = 0, InvalidSignature, + NoFailure, // TODO: Add other failure types } diff --git a/crates/sui-benchmark/src/workloads/workload_configuration.rs b/crates/sui-benchmark/src/workloads/workload_configuration.rs index 9cbd18800ead3..d6d9b2f753b81 100644 --- a/crates/sui-benchmark/src/workloads/workload_configuration.rs +++ b/crates/sui-benchmark/src/workloads/workload_configuration.rs @@ -18,9 +18,11 @@ use tracing::info; use super::adversarial::{AdversarialPayloadCfg, AdversarialWorkloadBuilder}; use super::expected_failure::{ExpectedFailurePayloadCfg, ExpectedFailureWorkloadBuilder}; +use super::randomized_transaction::RandomizedTransactionWorkloadBuilder; use super::randomness::RandomnessWorkloadBuilder; use super::shared_object_deletion::SharedCounterDeletionWorkloadBuilder; +#[derive(Debug)] pub struct WorkloadWeights { pub shared_counter: u32, pub transfer_object: u32, @@ -30,6 +32,7 @@ pub struct WorkloadWeights { pub adversarial: u32, pub expected_failure: u32, pub randomness: u32, + pub randomized_transaction: u32, } pub struct WorkloadConfig { @@ -69,6 +72,7 @@ impl WorkloadConfiguration { adversarial, expected_failure, randomness, + randomized_transaction, shared_counter_hotness_factor, num_shared_counters, shared_counter_max_tip, @@ -102,6 +106,7 @@ impl WorkloadConfiguration { adversarial: adversarial[i], expected_failure: expected_failure[i], randomness: randomness[i], + randomized_transaction: randomized_transaction[i], }, adversarial_cfg: AdversarialPayloadCfg::from_str(&adversarial_cfg[i]) .unwrap(), @@ -193,6 +198,13 @@ impl WorkloadConfiguration { }: WorkloadConfig, system_state_observer: Arc, ) -> Vec> { + tracing::info!( + "Workload Configuration weights {:?} target_qps: {:?} num_workers: {:?} duration: {:?}", + weights, + target_qps, + num_workers, + duration + ); let total_weight = weights.shared_counter + weights.shared_deletion + weights.transfer_object @@ -200,7 +212,8 @@ impl WorkloadConfiguration { + weights.batch_payment + weights.adversarial + weights.randomness - + weights.expected_failure; + + weights.expected_failure + + weights.randomized_transaction; let reference_gas_price = system_state_observer.state.borrow().reference_gas_price; let mut workload_builders = vec![]; let shared_workload = SharedCounterWorkloadBuilder::from( @@ -288,6 +301,16 @@ impl WorkloadConfiguration { group, ); workload_builders.push(expected_failure_workload); + let randomized_transaction_workload = RandomizedTransactionWorkloadBuilder::from( + weights.randomized_transaction as f32 / total_weight as f32, + target_qps, + num_workers, + in_flight_ratio, + reference_gas_price, + duration, + group, + ); + workload_builders.push(randomized_transaction_workload); workload_builders } diff --git a/crates/sui-benchmark/tests/simtest.rs b/crates/sui-benchmark/tests/simtest.rs index 88e4c5f7a3091..b3e748f21b198 100644 --- a/crates/sui-benchmark/tests/simtest.rs +++ b/crates/sui-benchmark/tests/simtest.rs @@ -558,7 +558,7 @@ mod test { let mut simulated_load_config = SimulatedLoadConfig::default(); { let mut rng = thread_rng(); - simulated_load_config.shared_counter_weight = if rng.gen_bool(0.5) { 5 } else { 50 }; + simulated_load_config.shared_counter_weight = if rng.gen_bool(0.5) { 5 } else { 5 }; simulated_load_config.num_shared_counters = match rng.gen_range(0..=2) { 0 => None, // shared_counter_hotness_factor is in play in this case. n => Some(n), @@ -571,7 +571,7 @@ mod test { info!("Simulated load config: {:?}", simulated_load_config); } - test_simulated_load_with_test_config(test_cluster, 50, simulated_load_config, None, None) + test_simulated_load_with_test_config(test_cluster, 180, simulated_load_config, None, None) .await; } @@ -937,6 +937,7 @@ mod test { shared_deletion_weight: u32, shared_counter_hotness_factor: u32, randomness_weight: u32, + randomized_transaction_weight: u32, num_shared_counters: Option, use_shared_counter_max_tip: bool, shared_counter_max_tip: u64, @@ -955,6 +956,7 @@ mod test { shared_deletion_weight: 1, shared_counter_hotness_factor: 50, randomness_weight: 1, + randomized_transaction_weight: 1, num_shared_counters: Some(1), use_shared_counter_max_tip: false, shared_counter_max_tip: 0, @@ -1013,7 +1015,7 @@ mod test { // The default test parameters are somewhat conservative in order to keep the running time // of the test reasonable in CI. - let target_qps = target_qps.unwrap_or(get_var("SIM_STRESS_TEST_QPS", 10)); + let target_qps = target_qps.unwrap_or(get_var("SIM_STRESS_TEST_QPS", 20)); let num_workers = num_workers.unwrap_or(get_var("SIM_STRESS_TEST_WORKERS", 10)); let in_flight_ratio = get_var("SIM_STRESS_TEST_IFR", 2); let batch_payment_size = get_var("SIM_BATCH_PAYMENT_SIZE", 15); @@ -1043,6 +1045,7 @@ mod test { randomness: config.randomness_weight, adversarial: adversarial_weight, expected_failure: config.expected_failure_weight, + randomized_transaction: config.randomized_transaction_weight, }; let workload_config = WorkloadConfig {