[forge] Split out test suites from main

There are a lot of tests and cruft in with the main forge cli

Split them out into suites

Test Plan:

running forge on PR succeeds

testsuite/forge-cli/src/suites/dag.rs

@@ -1,7 +1,10 @@
 // Copyright © Aptos Foundation
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::{changing_working_quorum_test_helper, wrap_with_realistic_env, TestCommand};
+use super::{
+    realistic_environment::wrap_with_realistic_env, ungrouped::changing_working_quorum_test_helper,
+};
+use crate::TestCommand;
 use aptos_forge::{
     success_criteria::{LatencyType, StateProgressThreshold, SuccessCriteria},
     EmitJobMode, EmitJobRequest, ForgeConfig,

testsuite/forge-cli/src/suites/db.rs

@@ -0,0 +1,53 @@
+use super::ungrouped::PROGRESS_THRESHOLD_20_6;
+use aptos_forge::{
+    args::TransactionTypeArg, success_criteria::SuccessCriteria, EmitJobMode, EmitJobRequest,
+    ForgeConfig,
+};
+use aptos_testcases::performance_test::PerformanceBenchmark;
+use std::{num::NonZeroUsize, path::PathBuf, sync::Arc};
+
+pub(crate) fn large_db_simple_test() -> ForgeConfig {
+    large_db_test(10, 500, 300, "10-validators".to_string())
+}
+
+pub(crate) fn large_db_test(
+    num_validators: usize,
+    target_tps: usize,
+    min_avg_tps: usize,
+    existing_db_tag: String,
+) -> ForgeConfig {
+    let config = ForgeConfig::default();
+    config
+        .with_initial_validator_count(NonZeroUsize::new(num_validators).unwrap())
+        .with_initial_fullnode_count(std::cmp::max(2, target_tps / 1000))
+        .add_network_test(PerformanceBenchmark)
+        .with_existing_db(existing_db_tag.clone())
+        .with_validator_override_node_config_fn(Arc::new(move |config, _| {
+            config.base.working_dir = Some(PathBuf::from("/opt/aptos/data/checkpoint"));
+        }))
+        .with_fullnode_override_node_config_fn(Arc::new(move |config, _| {
+            config.base.working_dir = Some(PathBuf::from("/opt/aptos/data/checkpoint"));
+        }))
+        .with_emit_job(
+            EmitJobRequest::default()
+                .mode(EmitJobMode::ConstTps { tps: target_tps })
+                .transaction_mix(vec![
+                    (TransactionTypeArg::CoinTransfer.materialize_default(), 75),
+                    (
+                        TransactionTypeArg::AccountGeneration.materialize_default(),
+                        20,
+                    ),
+                    (
+                        TransactionTypeArg::TokenV1NFTMintAndTransferSequential
+                            .materialize_default(),
+                        5,
+                    ),
+                ]),
+        )
+        .with_success_criteria(
+            SuccessCriteria::new(min_avg_tps)
+                .add_no_restarts()
+                .add_wait_for_catchup_s(30)
+                .add_chain_progress(PROGRESS_THRESHOLD_20_6.clone()),
+        )
+}

testsuite/forge-cli/src/suites/indexer.rs

@@ -0,0 +1,123 @@
+// Copyright © Aptos Foundation
+// Parts of the project are originally copyright © Meta Platforms, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{
+    realistic_environment::realistic_env_sweep_wrap, ungrouped::background_traffic_for_sweep,
+};
+use aptos_forge::{
+    args::TransactionTypeArg,
+    prometheus_metrics::LatencyBreakdownSlice,
+    success_criteria::{LatencyBreakdownThreshold, SuccessCriteria},
+    ForgeConfig,
+};
+use aptos_testcases::{
+    load_vs_perf_benchmark::{LoadVsPerfBenchmark, TransactionWorkload, Workloads},
+    performance_test::PerformanceBenchmark,
+};
+
+/// Attempts to match the test name to an indexer test
+pub fn get_indexer_test(test_name: &str) -> Option<ForgeConfig> {
+    let test = match test_name {
+        "indexer_test" => indexer_test(),
+        _ => return None, // The test name does not match an indexer test
+    };
+    Some(test)
+}
+
+/// Workload sweep with multiple stressful workloads for indexer
+fn indexer_test() -> ForgeConfig {
+    // Define all the workloads and their corresponding success criteria upfront
+    // The TransactionTypeArg is the workload per phase
+    // The structure of the success criteria is generally (min_tps, latencies...). See below for the exact definition.
+    // NOTES on tuning these criteria:
+    // * The blockchain's TPS criteria are generally lower than that of other tests. This is because we want to only capture indexer performance regressions. Other tests with higher TPS requirements
+    //   for the blockchain would catch an earlier regression
+    // * The indexer latencies are inter-component within the indexer stack, but not that of the e2e wall-clock time vs the blockchain
+    let workloads_and_criteria = vec![
+        (
+            TransactionWorkload::new(TransactionTypeArg::CoinTransfer, 20000),
+            (7000, 0.5, 1.0, 0.1),
+        ),
+        (
+            TransactionWorkload::new(TransactionTypeArg::NoOp, 20000).with_num_modules(100),
+            (8500, 0.5, 1.0, 0.1),
+        ),
+        (
+            TransactionWorkload::new(TransactionTypeArg::ModifyGlobalResource, 6000)
+                .with_transactions_per_account(1),
+            (2000, 0.5, 0.5, 0.1),
+        ),
+        (
+            TransactionWorkload::new(TransactionTypeArg::TokenV2AmbassadorMint, 20000)
+                .with_unique_senders(),
+            (2000, 1.0, 1.0, 0.5),
+        ),
+        (
+            TransactionWorkload::new(TransactionTypeArg::PublishPackage, 200)
+                .with_transactions_per_account(1),
+            (28, 0.5, 1.0, 0.1),
+        ),
+        (
+            TransactionWorkload::new(TransactionTypeArg::VectorPicture30k, 100),
+            (100, 0.5, 1.0, 0.1),
+        ),
+        (
+            TransactionWorkload::new(TransactionTypeArg::SmartTablePicture30KWith200Change, 100),
+            (100, 0.5, 1.0, 0.1),
+        ),
+        (
+            TransactionWorkload::new(
+                TransactionTypeArg::TokenV1NFTMintAndTransferSequential,
+                1000,
+            ),
+            (500, 0.5, 1.0, 0.1),
+        ),
+        (
+            TransactionWorkload::new(TransactionTypeArg::TokenV1FTMintAndTransfer, 1000),
+            (500, 0.5, 0.5, 0.1),
+        ),
+    ];
+    let num_sweep = workloads_and_criteria.len();
+
+    let workloads = Workloads::TRANSACTIONS(
+        workloads_and_criteria
+            .iter()
+            .map(|(w, _)| w.clone())
+            .collect(),
+    );
+    let criteria = workloads_and_criteria
+        .iter()
+        .map(|(_, c)| {
+            let (
+                min_tps,
+                indexer_fullnode_processed_batch,
+                indexer_cache_worker_processed_batch,
+                indexer_data_service_all_chunks_sent,
+            ) = c.to_owned();
+            SuccessCriteria::new(min_tps).add_latency_breakdown_threshold(
+                LatencyBreakdownThreshold::new_strict(vec![
+                    (
+                        LatencyBreakdownSlice::IndexerFullnodeProcessedBatch,
+                        indexer_fullnode_processed_batch,
+                    ),
+                    (
+                        LatencyBreakdownSlice::IndexerCacheWorkerProcessedBatch,
+                        indexer_cache_worker_processed_batch,
+                    ),
+                    (
+                        LatencyBreakdownSlice::IndexerDataServiceAllChunksSent,
+                        indexer_data_service_all_chunks_sent,
+                    ),
+                ]),
+            )
+        })
+        .collect::<Vec<_>>();
+
+    realistic_env_sweep_wrap(4, 4, LoadVsPerfBenchmark {
+        test: Box::new(PerformanceBenchmark),
+        workloads,
+        criteria,
+        background_traffic: background_traffic_for_sweep(num_sweep),
+    })
+}

testsuite/forge-cli/src/suites/land_blocking.rs

@@ -0,0 +1,47 @@
+use super::ungrouped::mixed_emit_job;
+use crate::{suites::realistic_environment::realistic_env_max_load_test, TestCommand};
+use aptos_forge::{success_criteria::SuccessCriteria, ForgeConfig};
+use aptos_testcases::{
+    compatibility_test::SimpleValidatorUpgrade, framework_upgrade::FrameworkUpgrade,
+};
+use std::{num::NonZeroUsize, sync::Arc, time::Duration};
+
+/// Attempts to match the test name to a land-blocking test
+pub(crate) fn get_land_blocking_test(
+    test_name: &str,
+    duration: Duration,
+    test_cmd: &TestCommand,
+) -> Option<ForgeConfig> {
+    let test = match test_name {
+        "land_blocking" | "realistic_env_max_load" => {
+            realistic_env_max_load_test(duration, test_cmd, 7, 5)
+        },
+        "compat" => compat(),
+        "framework_upgrade" => framework_upgrade(),
+        _ => return None, // The test name does not match a land-blocking test
+    };
+    Some(test)
+}
+
+pub(crate) fn compat() -> ForgeConfig {
+    ForgeConfig::default()
+        .with_initial_validator_count(NonZeroUsize::new(4).unwrap())
+        .add_network_test(SimpleValidatorUpgrade)
+        .with_success_criteria(SuccessCriteria::new(5000).add_wait_for_catchup_s(240))
+        .with_genesis_helm_config_fn(Arc::new(|helm_values| {
+            helm_values["chain"]["epoch_duration_secs"] =
+                SimpleValidatorUpgrade::EPOCH_DURATION_SECS.into();
+        }))
+}
+
+pub(crate) fn framework_upgrade() -> ForgeConfig {
+    ForgeConfig::default()
+        .with_initial_validator_count(NonZeroUsize::new(4).unwrap())
+        .add_network_test(FrameworkUpgrade)
+        .with_success_criteria(SuccessCriteria::new(5000).add_wait_for_catchup_s(240))
+        .with_genesis_helm_config_fn(Arc::new(|helm_values| {
+            helm_values["chain"]["epoch_duration_secs"] =
+                FrameworkUpgrade::EPOCH_DURATION_SECS.into();
+        }))
+        .with_emit_job(mixed_emit_job())
+}

testsuite/forge-cli/src/suites/mod.rs

@@ -2,3 +2,13 @@
 // SPDX-License-Identifier: Apache-2.0
 
 pub mod dag;
+pub mod db;
+pub mod indexer;
+pub mod land_blocking;
+pub mod multi_region;
+pub mod netbench;
+pub mod pfn;
+pub mod realistic_environment;
+pub mod realistic_workloads;
+pub mod state_sync;
+pub mod ungrouped;

testsuite/forge-cli/src/suites/multi_region.rs

@@ -0,0 +1,111 @@
+use crate::suites::{
+    state_sync::state_sync_config_execute_transactions,
+    ungrouped::{
+        PROGRESS_THRESHOLD_20_6, RELIABLE_PROGRESS_THRESHOLD, SYSTEM_12_CORES_10GB_THRESHOLD,
+    },
+};
+use aptos_forge::{
+    success_criteria::SuccessCriteria, EmitJobMode, EmitJobRequest, ForgeConfig, NetworkTest,
+};
+use aptos_testcases::{
+    modifiers::{ExecutionDelayConfig, ExecutionDelayTest},
+    multi_region_network_test::{
+        MultiRegionNetworkEmulationConfig, MultiRegionNetworkEmulationTest,
+    },
+    performance_test::PerformanceBenchmark,
+    three_region_simulation_test::ThreeRegionSameCloudSimulationTest,
+    CompositeNetworkTest,
+};
+use std::{num::NonZeroUsize, sync::Arc};
+
+/// Attempts to match the test name to a multi-region test
+pub(crate) fn get_multi_region_test(test_name: &str) -> Option<ForgeConfig> {
+    let test = match test_name {
+        "multiregion_benchmark_test" => multiregion_benchmark_test(),
+        "three_region_simulation" => three_region_simulation(),
+        "three_region_simulation_with_different_node_speed" => {
+            three_region_simulation_with_different_node_speed()
+        },
+        _ => return None, // The test name does not match a multi-region test
+    };
+    Some(test)
+}
+
+/// This test runs a network test in a real multi-region setup. It configures
+/// genesis and node helm values to enable certain configurations needed to run in
+/// the multiregion forge cluster.
+pub(crate) fn multiregion_benchmark_test() -> ForgeConfig {
+    ForgeConfig::default()
+        .with_initial_validator_count(NonZeroUsize::new(20).unwrap())
+        .add_network_test(PerformanceBenchmark)
+        .with_genesis_helm_config_fn(Arc::new(|helm_values| {
+            // Have single epoch change in land blocking
+            helm_values["chain"]["epoch_duration_secs"] = 300.into();
+
+            helm_values["genesis"]["multicluster"]["enabled"] = true.into();
+        }))
+        .with_multi_region_config()
+        .with_success_criteria(
+            SuccessCriteria::new(4500)
+                .add_no_restarts()
+                .add_wait_for_catchup_s(
+                    // Give at least 60s for catchup, give 10% of the run for longer durations.
+                    180,
+                )
+                .add_system_metrics_threshold(SYSTEM_12_CORES_10GB_THRESHOLD.clone())
+                .add_chain_progress(RELIABLE_PROGRESS_THRESHOLD.clone()),
+        )
+}
+
+pub(crate) fn three_region_simulation_with_different_node_speed() -> ForgeConfig {
+    ForgeConfig::default()
+        .with_initial_validator_count(NonZeroUsize::new(30).unwrap())
+        .with_initial_fullnode_count(30)
+        .with_emit_job(EmitJobRequest::default().mode(EmitJobMode::ConstTps { tps: 5000 }))
+        .add_network_test(CompositeNetworkTest::new(
+            ExecutionDelayTest {
+                add_execution_delay: ExecutionDelayConfig {
+                    inject_delay_node_fraction: 0.5,
+                    inject_delay_max_transaction_percentage: 40,
+                    inject_delay_per_transaction_ms: 2,
+                },
+            },
+            ThreeRegionSameCloudSimulationTest,
+        ))
+        .with_validator_override_node_config_fn(Arc::new(|config, _| {
+            config.api.failpoints_enabled = true;
+        }))
+        .with_fullnode_override_node_config_fn(Arc::new(|config, _| {
+            state_sync_config_execute_transactions(&mut config.state_sync);
+        }))
+        .with_success_criteria(
+            SuccessCriteria::new(1000)
+                .add_no_restarts()
+                .add_wait_for_catchup_s(240)
+                .add_chain_progress(PROGRESS_THRESHOLD_20_6.clone()),
+        )
+}
+
+pub(crate) fn three_region_simulation() -> ForgeConfig {
+    ForgeConfig::default()
+        .with_initial_validator_count(NonZeroUsize::new(12).unwrap())
+        .with_initial_fullnode_count(12)
+        .with_emit_job(EmitJobRequest::default().mode(EmitJobMode::ConstTps { tps: 5000 }))
+        .add_network_test(ThreeRegionSameCloudSimulationTest)
+        // TODO(rustielin): tune these success criteria after we have a better idea of the test behavior
+        .with_success_criteria(
+            SuccessCriteria::new(3000)
+                .add_no_restarts()
+                .add_wait_for_catchup_s(240)
+                .add_chain_progress(PROGRESS_THRESHOLD_20_6.clone()),
+        )
+}
+
+pub fn wrap_with_two_region_env<T: NetworkTest + 'static>(test: T) -> CompositeNetworkTest {
+    CompositeNetworkTest::new(
+        MultiRegionNetworkEmulationTest::new_with_config(
+            MultiRegionNetworkEmulationConfig::two_region(),
+        ),
+        test,
+    )
+}