From f3ef25102b9f81d1dce4103c00cfd9528e4ecf21 Mon Sep 17 00:00:00 2001 From: Igor Date: Fri, 22 Nov 2024 13:03:14 -0800 Subject: [PATCH 1/2] Calibration script for move e2e and single node perf benchmarks --- .../e2e-benchmark/data/calibration_values.tsv | 26 +++ aptos-move/e2e-benchmark/src/main.rs | 34 +--- testsuite/single_node_performance.py | 49 +---- .../single_node_performance_calibration.py | 173 ++++++++++++++++++ testsuite/single_node_performance_values.tsv | 44 +++++ 5 files changed, 249 insertions(+), 77 deletions(-) create mode 100644 aptos-move/e2e-benchmark/data/calibration_values.tsv create mode 100755 testsuite/single_node_performance_calibration.py create mode 100644 testsuite/single_node_performance_values.tsv diff --git a/aptos-move/e2e-benchmark/data/calibration_values.tsv b/aptos-move/e2e-benchmark/data/calibration_values.tsv new file mode 100644 index 0000000000000..47ae8b7258196 --- /dev/null +++ b/aptos-move/e2e-benchmark/data/calibration_values.tsv @@ -0,0 +1,26 @@ +Loop { loop_count: Some(100000), loop_type: NoOp } 60 0.955 1.074 41893.7 +Loop { loop_count: Some(10000), loop_type: Arithmetic } 60 0.965 1.078 25915.0 +CreateObjects { num_objects: 10, object_payload_size: 0 } 60 0.924 1.082 158.1 +CreateObjects { num_objects: 10, object_payload_size: 10240 } 60 0.951 1.118 9356.2 +CreateObjects { num_objects: 100, object_payload_size: 0 } 60 0.926 1.082 1574.2 +CreateObjects { num_objects: 100, object_payload_size: 10240 } 60 0.952 1.092 11541.9 +InitializeVectorPicture { length: 128 } 10 0.965 1.038 163.3 +VectorPicture { length: 128 } 10 0.938 1.060 48.8 +VectorPictureRead { length: 128 } 10 0.977 1.077 46.4 +InitializeVectorPicture { length: 30720 } 60 0.948 1.123 27893.4 +VectorPicture { length: 30720 } 60 0.931 1.125 6923.1 +VectorPictureRead { length: 30720 } 60 0.934 1.102 6923.1 +SmartTablePicture { length: 30720, num_points_per_txn: 200 } 60 0.952 1.109 43594.7 +SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 60 0.957 1.120 73865.4 +ResourceGroupsSenderWriteTag { string_length: 1024 } 60 0.934 1.134 15.0 +ResourceGroupsSenderMultiChange { string_length: 1024 } 60 0.929 1.122 32.3 +TokenV1MintAndTransferFT 60 0.958 1.093 385.2 +TokenV1MintAndTransferNFTSequential 60 0.973 1.139 588.1 +TokenV2AmbassadorMint { numbered: true } 60 0.960 1.141 512.5 +LiquidityPoolSwap { is_stable: true } 60 0.961 1.103 590.3 +LiquidityPoolSwap { is_stable: false } 60 0.954 1.134 552.2 +CoinInitAndMint 10 0.975 1.043 199.6 +FungibleAssetMint 10 0.954 1.038 236.3 +IncGlobalMilestoneAggV2 { milestone_every: 1 } 10 0.960 1.047 32.9 +IncGlobalMilestoneAggV2 { milestone_every: 2 } 10 0.971 1.066 18.1 +EmitEvents { count: 1000 } 10 0.969 1.052 8615.5 \ No newline at end of file diff --git a/aptos-move/e2e-benchmark/src/main.rs b/aptos-move/e2e-benchmark/src/main.rs index e00db94985007..252bf800bd79e 100644 --- a/aptos-move/e2e-benchmark/src/main.rs +++ b/aptos-move/e2e-benchmark/src/main.rs @@ -15,7 +15,7 @@ use aptos_transaction_generator_lib::{ use aptos_types::{account_address::AccountAddress, transaction::TransactionPayload}; use rand::{rngs::StdRng, SeedableRng}; use serde_json::json; -use std::{collections::HashMap, process::exit}; +use std::{collections::HashMap, fs, process::exit}; // bump after a bigger test or perf change, so you can easily distinguish runs // that are on top of this commit @@ -85,42 +85,14 @@ const ALLOWED_REGRESSION: f64 = 0.15; const ALLOWED_IMPROVEMENT: f64 = 0.15; const ABSOLUTE_BUFFER_US: f64 = 2.0; -const CALIBRATION_VALUES: &str = " -Loop { loop_count: Some(100000), loop_type: NoOp } 60 0.955 1.074 41893.7 -Loop { loop_count: Some(10000), loop_type: Arithmetic } 60 0.965 1.078 25915.0 -CreateObjects { num_objects: 10, object_payload_size: 0 } 60 0.924 1.082 158.1 -CreateObjects { num_objects: 10, object_payload_size: 10240 } 60 0.951 1.118 9356.2 -CreateObjects { num_objects: 100, object_payload_size: 0 } 60 0.926 1.082 1574.2 -CreateObjects { num_objects: 100, object_payload_size: 10240 } 60 0.952 1.092 11541.9 -InitializeVectorPicture { length: 128 } 10 0.965 1.038 163.3 -VectorPicture { length: 128 } 10 0.938 1.060 48.8 -VectorPictureRead { length: 128 } 10 0.977 1.077 46.4 -InitializeVectorPicture { length: 30720 } 60 0.948 1.123 27893.4 -VectorPicture { length: 30720 } 60 0.931 1.125 6923.1 -VectorPictureRead { length: 30720 } 60 0.934 1.102 6923.1 -SmartTablePicture { length: 30720, num_points_per_txn: 200 } 60 0.952 1.109 43594.7 -SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 60 0.957 1.120 73865.4 -ResourceGroupsSenderWriteTag { string_length: 1024 } 60 0.934 1.134 15.0 -ResourceGroupsSenderMultiChange { string_length: 1024 } 60 0.929 1.122 32.3 -TokenV1MintAndTransferFT 60 0.958 1.093 385.2 -TokenV1MintAndTransferNFTSequential 60 0.973 1.139 588.1 -TokenV2AmbassadorMint { numbered: true } 60 0.960 1.141 512.5 -LiquidityPoolSwap { is_stable: true } 60 0.961 1.103 590.3 -LiquidityPoolSwap { is_stable: false } 60 0.954 1.134 552.2 -CoinInitAndMint 10 0.975 1.043 199.6 -FungibleAssetMint 10 0.954 1.038 236.3 -IncGlobalMilestoneAggV2 { milestone_every: 1 } 10 0.960 1.047 32.9 -IncGlobalMilestoneAggV2 { milestone_every: 2 } 10 0.971 1.066 18.1 -EmitEvents { count: 1000 } 10 0.969 1.052 8615.5 -"; - struct CalibrationInfo { // count: usize, expected_time_micros: f64, } fn get_parsed_calibration_values() -> HashMap { - CALIBRATION_VALUES + let calibration_values = fs::read_to_string("aptos-move/e2e-benchmark/data/calibration_values.tsv").expect("Unable to read file"); + calibration_values .trim() .split('\n') .map(|line| { diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 8d78dc89d8b9d..171ff766a2a67 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -166,52 +166,9 @@ class RunGroupConfig: CALIBRATION_SEPARATOR = " " # transaction_type module_working_set_size executor_type count min_ratio max_ratio median -CALIBRATION = """ -no-op 1 VM 6 0.938 1.019 38925.3 -no-op 1000 VM 6 0.943 1.019 36444.6 -apt-fa-transfer 1 VM 6 0.927 1.018 26954.7 -apt-fa-transfer 1 NativeVM 6 0.927 1.018 35259.7 -account-generation 1 VM 6 0.96 1.02 20606.2 -account-generation 1 NativeVM 6 0.96 1.02 28216.2 -account-resource32-b 1 VM 6 0.94 1.026 34260.4 -modify-global-resource 1 VM 6 0.993 1.021 2260.5 -modify-global-resource 100 VM 6 0.982 1.02 33129.7 -publish-package 1 VM 6 0.983 1.012 1672.6 -mix_publish_transfer 1 VM 6 0.972 1.044 20832.8 -batch100-transfer 1 VM 6 0.953 1.024 645.1 -batch100-transfer 1 NativeVM 6 0.953 1.024 1437.0 -vector-picture30k 1 VM 6 0.992 1.039 103.6 -vector-picture30k 100 VM 6 0.913 1.015 1831.5 -smart-table-picture30-k-with200-change 1 VM 6 0.976 1.034 16.1 -smart-table-picture30-k-with200-change 100 VM 6 0.985 1.018 212.9 -modify-global-resource-agg-v2 1 VM 6 0.976 1.035 33992.5 -modify-global-flag-agg-v2 1 VM 6 0.986 1.016 4224 -modify-global-bounded-agg-v2 1 VM 6 0.964 1.047 7661.6 -modify-global-milestone-agg-v2 1 VM 6 0.973 1.017 25187.1 -resource-groups-global-write-tag1-kb 1 VM 6 0.989 1.03 9215.7 -resource-groups-global-write-and-read-tag1-kb 1 VM 6 0.982 1.018 5538.3 -resource-groups-sender-write-tag1-kb 1 VM 6 0.985 1.059 20084.2 -resource-groups-sender-multi-change1-kb 1 VM 6 0.968 1.034 16400.4 -token-v1ft-mint-and-transfer 1 VM 6 0.987 1.022 1156.3 -token-v1ft-mint-and-transfer 100 VM 6 0.964 1.024 17842.6 -token-v1nft-mint-and-transfer-sequential 1 VM 6 0.984 1.017 735.7 -token-v1nft-mint-and-transfer-sequential 100 VM 6 0.966 1.017 12819.7 -coin-init-and-mint 1 VM 6 0.95 1.024 26906.4 -coin-init-and-mint 100 VM 6 0.985 1.022 22312.6 -fungible-asset-mint 1 VM 6 0.955 1.013 23001.6 -fungible-asset-mint 100 VM 6 0.955 1.015 19973.5 -no-op5-signers 1 VM 6 0.934 1.016 38708.6 -token-v2-ambassador-mint 1 VM 6 0.975 1.008 15179.3 -token-v2-ambassador-mint 100 VM 6 0.985 1.007 15150.8 -liquidity-pool-swap 1 VM 6 0.987 1.018 805.5 -liquidity-pool-swap 100 VM 6 0.993 1.02 11156.3 -liquidity-pool-swap-stable 1 VM 6 0.985 1.017 778.7 -liquidity-pool-swap-stable 100 VM 6 0.982 1.009 11056.6 -deserialize-u256 1 VM 6 0.968 1.026 36444.6 -no-op-fee-payer 1 VM 6 0.994 1.026 2046 -no-op-fee-payer 100 VM 6 0.96 1.014 32866.5 -simple-script 1 VM 6 0.941 1.012 38206.1 -""" +with open('testsuite/single_node_performance_values.tsv', 'r') as file: + CALIBRATION = file.read() + # when adding a new test, add estimated expected_tps to it, as well as waived=True. # And then after a day or two - add calibration result for it above, removing expected_tps/waived fields. diff --git a/testsuite/single_node_performance_calibration.py b/testsuite/single_node_performance_calibration.py new file mode 100755 index 0000000000000..e33d60e427483 --- /dev/null +++ b/testsuite/single_node_performance_calibration.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python + +import argparse +import requests + +def humio_secret(): + import subprocess + return subprocess.run(["gcloud", "secrets", "versions", "access", "--secret=ci_humio_read_token", "--project=aptos-shared-secrets", "latest"], capture_output=True).stdout.decode("utf-8") + + +def humio_url(): + return "https://cloud.us.humio.com/api/v1/repositories/github/query" + +def parse_args(): + parser = argparse.ArgumentParser(description='Benchmark calibration tools') + + parser.add_argument( + '--branch', + type=str, + help='Optional branch, if passed - only looks at results run on that branch through adhoc runs', + ) + + parser.add_argument( + '--move-e2e', + action='store_true', + help='Calibrate move e2e test', + ) + + return parser.parse_args() + +def query_humio(query_string, time_interval="5d"): + query = { + "queryString": query_string, + "start": time_interval, + } + + secret = humio_secret() + + resp = requests.post( + url=humio_url(), + json=query, + headers={ + "Authorization": f"Bearer {secret}", + "Content-Type": "application/json", + }, + ) + + return resp.text.strip() + +def main(): + args = parse_args() + + if args.move_e2e: + prefix = ( + """ + github.job.name = "single-node-performance" + | github.workflow.head_branch = "{branch}" + | "grep_json_aptos_move_vm_perf" + | parseJson(message) + """.format( + branch=args.branch + ) + if args.branch is not None + else """ + github.job.name = "execution-performance / single-node-performance" + | github.workflow.head_branch = "main" + | "grep_json_aptos_move_vm_perf" + | parseJson(message) + """ + ) + + query_string = ( + prefix + + """ + | groupBy([test_index, transaction_type], function=[count(as="count"), avg(expected_wall_time_us, as="expected"), avg(wall_time_us, as="avg_wall_time_us"), min(wall_time_us, as="min_wall_time_us"), max(wall_time_us, as="max_wall_time_us"), percentile(field=wall_time_us, accuracy=0.001, percentiles=[50])]) + | min_ratio := min_wall_time_us / _50 + | avg_ratio := avg_wall_time_us / _50 + | max_ratio := max_wall_time_us / _50 + | offset_avg_from_expected := _50 / expected + | format("%.1f", field=_50, as="median") + | format("%.1f", field=avg_wall_time_us, as="avg_wall_time_us") + | format("%.1f", field=min_wall_time_us, as="min_wall_time_us") + | format("%.1f", field=max_wall_time_us, as="max_wall_time_us") + | format("%.3f", field=min_ratio, as="min_ratio") + | format("%.3f", field=max_ratio, as="max_ratio") + | format("%.3f", field=offset_avg_from_expected, as="offset_median_from_expected") + | table([transaction_type, count, min_ratio, max_ratio, median, expected], sortby=test_index, reverse=false) + """ + ) + + columns = ["transaction_type", "count", "min_ratio", "max_ratio", "median"] + + def split_line(line): + line = line.strip() + if "}" in line: + parts = line.split("}") + res = [parts[0] + "}"] + parts[1].split(", ")[1:] + return res + else: + return line.split(", ") + output_file_name = "aptos-move/e2e-benchmark/data/calibration_values.tsv" + + else: + prefix = ( + """ + github.job.name = "single-node-performance" + | github.workflow.head_branch = "{branch}" + | "grep_json_single_node_perf" + | parseJson(message) + | source = "ADHOC" + | code_perf_version = "v8" + """.format( + branch=args.branch + ) + if args.branch is not None + else """ + github.job.name = "execution-performance / single-node-performance" + | github.workflow.head_branch = "main" + | "grep_json_single_node_perf" + | parseJson(message) + """ + ) + + query_string = ( + prefix + + """ + | groupBy([test_index, transaction_type, module_working_set_size, executor_type, code_perf_version], function=[count(as="count"), avg(expected_tps, as="expected"), avg(tps, as="avg_tps"), min(tps, as="min_tps"), max(tps, as="max_tps"), percentile(field=tps, accuracy=0.001, percentiles=[50])]) + | min_ratio := min_tps / _50 + | avg_ratio := avg_tps / _50 + | max_ratio := max_tps / _50 + | offset_avg_from_expected := _50 / expected + | format("%.1f", field=_50, as="median") + | format("%.1f", field=avg_tps, as="avg_tps") + | format("%.1f", field=min_tps, as="min_tps") + | format("%.1f", field=max_tps, as="max_tps") + | format("%.3f", field=min_ratio, as="min_ratio") + | format("%.3f", field=max_ratio, as="max_ratio") + | format("%.3f", field=offset_avg_from_expected, as="offset_median_from_expected") + | table([transaction_type, module_working_set_size, executor_type, count, min_ratio, max_ratio, median], sortby=test_index, reverse=false) + """ + ) + + columns = [ + "transaction_type", + "module_working_set_size", + "executor_type", + "count", + "min_ratio", + "max_ratio", + "median", + ] + + def split_line(line): + return line.strip().split(", ") + + output_file_name = "testsuite/single_node_performance_values.tsv" + + response_text = query_humio(query_string) + + parsed = [ + {(parts := key_value.split("->"))[0]: parts[1] for key_value in split_line(line)} + for line in response_text.split("\n") + ] + + with open(output_file_name, "w") as f: + for line in parsed: + f.write("\t".join([line[column] for column in columns])) + f.write("\n") + + print(f"Written to {output_file_name}") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/testsuite/single_node_performance_values.tsv b/testsuite/single_node_performance_values.tsv new file mode 100644 index 0000000000000..57d7be160aca6 --- /dev/null +++ b/testsuite/single_node_performance_values.tsv @@ -0,0 +1,44 @@ +no-op 1 VM 6 0.938 1.019 38925.3 +no-op 1000 VM 6 0.943 1.019 36444.6 +apt-fa-transfer 1 VM 6 0.927 1.018 26954.7 +apt-fa-transfer 1 NativeVM 6 0.927 1.018 35259.7 +account-generation 1 VM 6 0.96 1.02 20606.2 +account-generation 1 NativeVM 6 0.96 1.02 28216.2 +account-resource32-b 1 VM 6 0.94 1.026 34260.4 +modify-global-resource 1 VM 6 0.993 1.021 2260.5 +modify-global-resource 100 VM 6 0.982 1.02 33129.7 +publish-package 1 VM 6 0.983 1.012 1672.6 +mix_publish_transfer 1 VM 6 0.972 1.044 20832.8 +batch100-transfer 1 VM 6 0.953 1.024 645.1 +batch100-transfer 1 NativeVM 6 0.953 1.024 1437.0 +vector-picture30k 1 VM 6 0.992 1.039 103.6 +vector-picture30k 100 VM 6 0.913 1.015 1831.5 +smart-table-picture30-k-with200-change 1 VM 6 0.976 1.034 16.1 +smart-table-picture30-k-with200-change 100 VM 6 0.985 1.018 212.9 +modify-global-resource-agg-v2 1 VM 6 0.976 1.035 33992.5 +modify-global-flag-agg-v2 1 VM 6 0.986 1.016 4224 +modify-global-bounded-agg-v2 1 VM 6 0.964 1.047 7661.6 +modify-global-milestone-agg-v2 1 VM 6 0.973 1.017 25187.1 +resource-groups-global-write-tag1-kb 1 VM 6 0.989 1.03 9215.7 +resource-groups-global-write-and-read-tag1-kb 1 VM 6 0.982 1.018 5538.3 +resource-groups-sender-write-tag1-kb 1 VM 6 0.985 1.059 20084.2 +resource-groups-sender-multi-change1-kb 1 VM 6 0.968 1.034 16400.4 +token-v1ft-mint-and-transfer 1 VM 6 0.987 1.022 1156.3 +token-v1ft-mint-and-transfer 100 VM 6 0.964 1.024 17842.6 +token-v1nft-mint-and-transfer-sequential 1 VM 6 0.984 1.017 735.7 +token-v1nft-mint-and-transfer-sequential 100 VM 6 0.966 1.017 12819.7 +coin-init-and-mint 1 VM 6 0.95 1.024 26906.4 +coin-init-and-mint 100 VM 6 0.985 1.022 22312.6 +fungible-asset-mint 1 VM 6 0.955 1.013 23001.6 +fungible-asset-mint 100 VM 6 0.955 1.015 19973.5 +no-op5-signers 1 VM 6 0.934 1.016 38708.6 +token-v2-ambassador-mint 1 VM 6 0.975 1.008 15179.3 +token-v2-ambassador-mint 100 VM 6 0.985 1.007 15150.8 +liquidity-pool-swap 1 VM 6 0.987 1.018 805.5 +liquidity-pool-swap 100 VM 6 0.993 1.02 11156.3 +liquidity-pool-swap-stable 1 VM 6 0.985 1.017 778.7 +liquidity-pool-swap-stable 100 VM 6 0.982 1.009 11056.6 +deserialize-u256 1 VM 6 0.968 1.026 36444.6 +no-op-fee-payer 1 VM 6 0.994 1.026 2046 +no-op-fee-payer 100 VM 6 0.96 1.014 32866.5 +simple-script 1 VM 6 0.941 1.012 38206.1 \ No newline at end of file From e4d29e8afcfed4ab0fb493f8889a6832bf65f3b8 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 2 Dec 2024 12:17:44 -0800 Subject: [PATCH 2/2] recalibrated values from script --- .../e2e-benchmark/data/calibration_values.tsv | 52 +++++------ aptos-move/e2e-benchmark/src/main.rs | 4 +- testsuite/forge.py | 9 +- testsuite/replay-verify/main.py | 31 +++++-- testsuite/single_node_performance.py | 20 ++-- .../single_node_performance_calibration.py | 53 ++++++++--- testsuite/single_node_performance_values.tsv | 92 ++++++++++--------- 7 files changed, 159 insertions(+), 102 deletions(-) diff --git a/aptos-move/e2e-benchmark/data/calibration_values.tsv b/aptos-move/e2e-benchmark/data/calibration_values.tsv index 47ae8b7258196..0c373cd15b35f 100644 --- a/aptos-move/e2e-benchmark/data/calibration_values.tsv +++ b/aptos-move/e2e-benchmark/data/calibration_values.tsv @@ -1,26 +1,26 @@ -Loop { loop_count: Some(100000), loop_type: NoOp } 60 0.955 1.074 41893.7 -Loop { loop_count: Some(10000), loop_type: Arithmetic } 60 0.965 1.078 25915.0 -CreateObjects { num_objects: 10, object_payload_size: 0 } 60 0.924 1.082 158.1 -CreateObjects { num_objects: 10, object_payload_size: 10240 } 60 0.951 1.118 9356.2 -CreateObjects { num_objects: 100, object_payload_size: 0 } 60 0.926 1.082 1574.2 -CreateObjects { num_objects: 100, object_payload_size: 10240 } 60 0.952 1.092 11541.9 -InitializeVectorPicture { length: 128 } 10 0.965 1.038 163.3 -VectorPicture { length: 128 } 10 0.938 1.060 48.8 -VectorPictureRead { length: 128 } 10 0.977 1.077 46.4 -InitializeVectorPicture { length: 30720 } 60 0.948 1.123 27893.4 -VectorPicture { length: 30720 } 60 0.931 1.125 6923.1 -VectorPictureRead { length: 30720 } 60 0.934 1.102 6923.1 -SmartTablePicture { length: 30720, num_points_per_txn: 200 } 60 0.952 1.109 43594.7 -SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 60 0.957 1.120 73865.4 -ResourceGroupsSenderWriteTag { string_length: 1024 } 60 0.934 1.134 15.0 -ResourceGroupsSenderMultiChange { string_length: 1024 } 60 0.929 1.122 32.3 -TokenV1MintAndTransferFT 60 0.958 1.093 385.2 -TokenV1MintAndTransferNFTSequential 60 0.973 1.139 588.1 -TokenV2AmbassadorMint { numbered: true } 60 0.960 1.141 512.5 -LiquidityPoolSwap { is_stable: true } 60 0.961 1.103 590.3 -LiquidityPoolSwap { is_stable: false } 60 0.954 1.134 552.2 -CoinInitAndMint 10 0.975 1.043 199.6 -FungibleAssetMint 10 0.954 1.038 236.3 -IncGlobalMilestoneAggV2 { milestone_every: 1 } 10 0.960 1.047 32.9 -IncGlobalMilestoneAggV2 { milestone_every: 2 } 10 0.971 1.066 18.1 -EmitEvents { count: 1000 } 10 0.969 1.052 8615.5 \ No newline at end of file +Loop { loop_count: Some(100000), loop_type: NoOp } 60 0.960 1.119 42122.6 +Loop { loop_count: Some(10000), loop_type: Arithmetic } 60 0.956 1.074 26240.7 +CreateObjects { num_objects: 10, object_payload_size: 0 } 60 0.938 1.168 156.6 +CreateObjects { num_objects: 10, object_payload_size: 10240 } 60 0.924 1.086 9713.2 +CreateObjects { num_objects: 100, object_payload_size: 0 } 60 0.922 1.275 1577.0 +CreateObjects { num_objects: 100, object_payload_size: 10240 } 60 0.935 1.070 11728.7 +InitializeVectorPicture { length: 128 } 60 0.926 1.069 169.4 +VectorPicture { length: 128 } 60 0.908 1.075 50.2 +VectorPictureRead { length: 128 } 60 0.919 1.059 48.0 +InitializeVectorPicture { length: 30720 } 60 0.939 1.127 28404.4 +VectorPicture { length: 30720 } 60 0.936 1.095 6935.6 +VectorPictureRead { length: 30720 } 60 0.939 1.093 6948.2 +SmartTablePicture { length: 30720, num_points_per_txn: 200 } 60 0.947 1.080 43673.3 +SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 60 0.947 1.111 74145.8 +ResourceGroupsSenderWriteTag { string_length: 1024 } 60 0.918 1.075 15.8 +ResourceGroupsSenderMultiChange { string_length: 1024 } 60 0.909 1.169 32.9 +TokenV1MintAndTransferFT 60 0.953 1.069 384.6 +TokenV1MintAndTransferNFTSequential 60 0.938 1.064 600.3 +TokenV2AmbassadorMint { numbered: true } 60 0.951 1.057 516.6 +LiquidityPoolSwap { is_stable: true } 60 0.961 1.139 582.6 +LiquidityPoolSwap { is_stable: false } 60 0.929 1.099 563.0 +CoinInitAndMint 60 0.928 1.130 205.0 +FungibleAssetMint 60 0.930 1.098 235.8 +IncGlobalMilestoneAggV2 { milestone_every: 1 } 60 0.914 1.051 33.5 +IncGlobalMilestoneAggV2 { milestone_every: 2 } 60 0.914 1.105 19.0 +EmitEvents { count: 1000 } 60 0.937 1.158 8818.7 diff --git a/aptos-move/e2e-benchmark/src/main.rs b/aptos-move/e2e-benchmark/src/main.rs index 252bf800bd79e..fd6365284d56d 100644 --- a/aptos-move/e2e-benchmark/src/main.rs +++ b/aptos-move/e2e-benchmark/src/main.rs @@ -91,7 +91,9 @@ struct CalibrationInfo { } fn get_parsed_calibration_values() -> HashMap { - let calibration_values = fs::read_to_string("aptos-move/e2e-benchmark/data/calibration_values.tsv").expect("Unable to read file"); + let calibration_values = + fs::read_to_string("aptos-move/e2e-benchmark/data/calibration_values.tsv") + .expect("Unable to read file"); calibration_values .trim() .split('\n') diff --git a/testsuite/forge.py b/testsuite/forge.py index f39f36c08f6a3..dcf15e3e3233b 100644 --- a/testsuite/forge.py +++ b/testsuite/forge.py @@ -619,8 +619,9 @@ def format_pre_comment(context: ForgeContext) -> str: context.forge_namespace, ) - return textwrap.dedent( - f""" + return ( + textwrap.dedent( + f""" ### Forge is running suite `{context.forge_test_suite}` on {get_testsuite_images(context)} * [Grafana dashboard (auto-refresh)]({dashboard_link}) * [Humio Logs]({humio_logs_link}) @@ -628,7 +629,9 @@ def format_pre_comment(context: ForgeContext) -> str: * [Validator CPU Profile]({validator_cpu_profile_link}) * [Fullnode CPU Profile]({fullnode_cpu_profile_link}) """ - ).lstrip() + format_github_info(context) + ).lstrip() + + format_github_info(context) + ) def format_comment(context: ForgeContext, result: ForgeResult) -> str: diff --git a/testsuite/replay-verify/main.py b/testsuite/replay-verify/main.py index e8b29e363b4ee..e3cb4acc0b26d 100644 --- a/testsuite/replay-verify/main.py +++ b/testsuite/replay-verify/main.py @@ -28,6 +28,7 @@ REPLAY_CONCURRENCY_LEVEL = 1 + class Network(Enum): TESTNET = 1 MAINNET = 2 @@ -241,6 +242,7 @@ def get_pod_status(self): def get_humio_log_link(self): return construct_humio_url(self.label, self.name, self.start_time, time.time()) + class ReplayConfig: def __init__(self, network): if network == Network.TESTNET: @@ -253,9 +255,10 @@ def __init__(self, network): self.concurrent_replayer = 18 self.pvc_number = 8 self.min_range_size = 10_000 - self.range_size = 2_000_000 + self.range_size = 2_000_000 self.timeout_secs = 400 + class TaskStats: def __init__(self, name): self.name = name @@ -308,7 +311,7 @@ def __init__( self.image = image self.pvcs = [] self.config = replay_config - + def __str__(self): return f"""ReplayScheduler: id: {self.id} @@ -360,7 +363,11 @@ def create_pvc_from_snapshot(self): else MAINNET_SNAPSHOT_NAME ) pvcs = create_pvcs_from_snapshot( - self.id, snapshot_name, self.namespace, self.config.pvc_number, self.get_label() + self.id, + snapshot_name, + self.namespace, + self.config.pvc_number, + self.get_label(), ) assert len(pvcs) == self.config.pvc_number, "failed to create all pvcs" self.pvcs = pvcs @@ -504,12 +511,16 @@ def get_image(image_tag=None): shell = forge.LocalShell() git = forge.Git(shell) image_name = "tools" - default_latest_image = forge.find_recent_images( - shell, - git, - 1, - image_name=image_name, - )[0] if image_tag is None else image_tag + default_latest_image = ( + forge.find_recent_images( + shell, + git, + 1, + image_name=image_name, + )[0] + if image_tag is None + else image_tag + ) full_image = f"{forge.GAR_REPO_NAME}/{image_name}:{default_latest_image}" return full_image @@ -546,7 +557,7 @@ def print_logs(failed_workpod_logs, txn_mismatch_logs): range_size=range_size, image=image, replay_config=config, - network= network, + network=network, namespace=args.namespace, ) logger.info(f"scheduler: {scheduler}") diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 171ff766a2a67..69a571cbadbc6 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -189,7 +189,7 @@ class RunGroupConfig: RunGroupConfig(key=RunGroupKey("mix_publish_transfer"), key_extra=RunGroupKeyExtra( transaction_type_override="publish-package apt-fa-transfer", transaction_weights_override="1 100", - ), included_in=LAND_BLOCKING_AND_C, waived=True), + ), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(key=RunGroupKey("batch100-transfer"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(key=RunGroupKey("batch100-transfer", executor_type="NativeVM"), included_in=Flow.CONTINUOUS), @@ -251,7 +251,7 @@ class RunGroupConfig: # fee payer sequentializes transactions today. in these tests module publisher is the fee payer, so larger number of modules tests throughput with multiple fee payers RunGroupConfig(key=RunGroupKey("no-op-fee-payer"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(key=RunGroupKey("no-op-fee-payer", module_working_set_size=DEFAULT_MODULE_WORKING_SET_SIZE), included_in=Flow.CONTINUOUS), - RunGroupConfig(key=RunGroupKey("simple-script"), included_in=LAND_BLOCKING_AND_C, waived=True), + RunGroupConfig(key=RunGroupKey("simple-script"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True), RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True), @@ -1024,16 +1024,24 @@ def print_table( """If you expect your PR to change the performance, you need to recalibrate the values. To do so, you should run the test on your branch 6 times (https://github.com/aptos-labs/aptos-core/actions/workflows/workflow-run-execution-performance.yaml ; remember to select CONTINUOUS). -Then go to Humio calibration link (https://gist.github.com/igor-aptos/7b12ca28de03894cddda8e415f37889e), -update it to your branch, and export values as CSV, and then open and copy values inside -testsuite/single_node_performance.py testsuite), and add Blockchain oncall as the reviewer. +Then run the script locally `./testsuite/single_node_performance_calibration.py --branch=YOUR_BRANCH` to update calibration values +and add Blockchain oncall as the reviewer. """ ) exit(1) if move_e2e_benchmark_failed: print( - "Move e2e benchmark failed, failing the job. See logs at the beginning for more details." + """ +Move e2e benchmark failed, failing the job. See logs at the beginning for more details. + +If you expect your PR to change the performance, you need to recalibrate the values. +To do so, you should run the test on your branch 6 times +(https://github.com/aptos-labs/aptos-core/actions/workflows/workflow-run-execution-performance.yaml ; remember to select CONTINUOUS, +and don't select to skip move-only e2e tests). +Then run the script locally `./testsuite/single_node_performance_calibration.py --branch=YOUR_BRANCH --move-e2e` to update calibration values +and add Blockchain oncall as the reviewer. +""" ) exit(1) diff --git a/testsuite/single_node_performance_calibration.py b/testsuite/single_node_performance_calibration.py index e33d60e427483..9a5d8ae705988 100755 --- a/testsuite/single_node_performance_calibration.py +++ b/testsuite/single_node_performance_calibration.py @@ -3,32 +3,54 @@ import argparse import requests + def humio_secret(): + print( + "trying to get a humio secret from gcloud. if it asks for a password, abort and run `gcloud auth login --update-adc` first" + ) import subprocess - return subprocess.run(["gcloud", "secrets", "versions", "access", "--secret=ci_humio_read_token", "--project=aptos-shared-secrets", "latest"], capture_output=True).stdout.decode("utf-8") + + return subprocess.run( + [ + "gcloud", + "secrets", + "versions", + "access", + "--secret=ci_humio_read_token", + "--project=aptos-shared-secrets", + "latest", + ], + capture_output=True, + ).stdout.decode("utf-8") def humio_url(): return "https://cloud.us.humio.com/api/v1/repositories/github/query" + def parse_args(): - parser = argparse.ArgumentParser(description='Benchmark calibration tools') + parser = argparse.ArgumentParser(description="Benchmark calibration tools") parser.add_argument( - '--branch', + "--branch", type=str, - help='Optional branch, if passed - only looks at results run on that branch through adhoc runs', + help="Optional branch, if passed - only looks at results run on that branch through adhoc runs", ) parser.add_argument( - '--move-e2e', - action='store_true', - help='Calibrate move e2e test', + "--move-e2e", + action="store_true", + help="Calibrate move e2e test", + ) + + parser.add_argument( + "--time-interval", default="5d", help="Time interval to look at humio for" ) return parser.parse_args() -def query_humio(query_string, time_interval="5d"): + +def query_humio(query_string, time_interval): query = { "queryString": query_string, "start": time_interval, @@ -47,6 +69,7 @@ def query_humio(query_string, time_interval="5d"): return resp.text.strip() + def main(): args = parse_args() @@ -57,6 +80,7 @@ def main(): | github.workflow.head_branch = "{branch}" | "grep_json_aptos_move_vm_perf" | parseJson(message) + | code_perf_version = "v1" """.format( branch=args.branch ) @@ -98,6 +122,7 @@ def split_line(line): return res else: return line.split(", ") + output_file_name = "aptos-move/e2e-benchmark/data/calibration_values.tsv" else: @@ -155,10 +180,13 @@ def split_line(line): output_file_name = "testsuite/single_node_performance_values.tsv" - response_text = query_humio(query_string) + response_text = query_humio(query_string, time_interval=args.time_interval) parsed = [ - {(parts := key_value.split("->"))[0]: parts[1] for key_value in split_line(line)} + { + (parts := key_value.split("->"))[0]: parts[1] + for key_value in split_line(line) + } for line in response_text.split("\n") ] @@ -169,5 +197,6 @@ def split_line(line): print(f"Written to {output_file_name}") -if __name__ == '__main__': - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/testsuite/single_node_performance_values.tsv b/testsuite/single_node_performance_values.tsv index 57d7be160aca6..49145e221deac 100644 --- a/testsuite/single_node_performance_values.tsv +++ b/testsuite/single_node_performance_values.tsv @@ -1,44 +1,48 @@ -no-op 1 VM 6 0.938 1.019 38925.3 -no-op 1000 VM 6 0.943 1.019 36444.6 -apt-fa-transfer 1 VM 6 0.927 1.018 26954.7 -apt-fa-transfer 1 NativeVM 6 0.927 1.018 35259.7 -account-generation 1 VM 6 0.96 1.02 20606.2 -account-generation 1 NativeVM 6 0.96 1.02 28216.2 -account-resource32-b 1 VM 6 0.94 1.026 34260.4 -modify-global-resource 1 VM 6 0.993 1.021 2260.5 -modify-global-resource 100 VM 6 0.982 1.02 33129.7 -publish-package 1 VM 6 0.983 1.012 1672.6 -mix_publish_transfer 1 VM 6 0.972 1.044 20832.8 -batch100-transfer 1 VM 6 0.953 1.024 645.1 -batch100-transfer 1 NativeVM 6 0.953 1.024 1437.0 -vector-picture30k 1 VM 6 0.992 1.039 103.6 -vector-picture30k 100 VM 6 0.913 1.015 1831.5 -smart-table-picture30-k-with200-change 1 VM 6 0.976 1.034 16.1 -smart-table-picture30-k-with200-change 100 VM 6 0.985 1.018 212.9 -modify-global-resource-agg-v2 1 VM 6 0.976 1.035 33992.5 -modify-global-flag-agg-v2 1 VM 6 0.986 1.016 4224 -modify-global-bounded-agg-v2 1 VM 6 0.964 1.047 7661.6 -modify-global-milestone-agg-v2 1 VM 6 0.973 1.017 25187.1 -resource-groups-global-write-tag1-kb 1 VM 6 0.989 1.03 9215.7 -resource-groups-global-write-and-read-tag1-kb 1 VM 6 0.982 1.018 5538.3 -resource-groups-sender-write-tag1-kb 1 VM 6 0.985 1.059 20084.2 -resource-groups-sender-multi-change1-kb 1 VM 6 0.968 1.034 16400.4 -token-v1ft-mint-and-transfer 1 VM 6 0.987 1.022 1156.3 -token-v1ft-mint-and-transfer 100 VM 6 0.964 1.024 17842.6 -token-v1nft-mint-and-transfer-sequential 1 VM 6 0.984 1.017 735.7 -token-v1nft-mint-and-transfer-sequential 100 VM 6 0.966 1.017 12819.7 -coin-init-and-mint 1 VM 6 0.95 1.024 26906.4 -coin-init-and-mint 100 VM 6 0.985 1.022 22312.6 -fungible-asset-mint 1 VM 6 0.955 1.013 23001.6 -fungible-asset-mint 100 VM 6 0.955 1.015 19973.5 -no-op5-signers 1 VM 6 0.934 1.016 38708.6 -token-v2-ambassador-mint 1 VM 6 0.975 1.008 15179.3 -token-v2-ambassador-mint 100 VM 6 0.985 1.007 15150.8 -liquidity-pool-swap 1 VM 6 0.987 1.018 805.5 -liquidity-pool-swap 100 VM 6 0.993 1.02 11156.3 -liquidity-pool-swap-stable 1 VM 6 0.985 1.017 778.7 -liquidity-pool-swap-stable 100 VM 6 0.982 1.009 11056.6 -deserialize-u256 1 VM 6 0.968 1.026 36444.6 -no-op-fee-payer 1 VM 6 0.994 1.026 2046 -no-op-fee-payer 100 VM 6 0.96 1.014 32866.5 -simple-script 1 VM 6 0.941 1.012 38206.1 \ No newline at end of file +no-op 1 VM 60 0.812 1.022 38853.0 +no-op 1000 VM 60 0.820 1.027 36792.8 +apt-fa-transfer 1 VM 60 0.831 1.028 27197.8 +apt-fa-transfer 1 NativeVM 60 0.829 1.158 32347.9 +account-generation 1 VM 60 0.776 1.035 20568.6 +account-generation 1 NativeVM 60 0.791 1.126 28249.8 +account-resource32-b 1 VM 60 0.819 1.027 34193.3 +modify-global-resource 1 VM 60 0.932 1.015 2273.4 +modify-global-resource 100 VM 60 0.838 1.019 33327.8 +publish-package 1 VM 60 0.887 1.014 1666.6 +mix_publish_transfer 1 VM 60 0.835 1.035 21291.6 +batch100-transfer 1 VM 60 0.794 1.041 634.6 +batch100-transfer 1 NativeVM 60 0.776 1.157 1387.1 +vector-picture30k 1 VM 60 0.960 1.051 102.5 +vector-picture30k 100 VM 60 0.754 1.051 1855.3 +smart-table-picture30-k-with200-change 1 VM 60 0.922 1.047 16.1 +smart-table-picture30-k-with200-change 100 VM 60 0.924 1.089 209.5 +modify-global-resource-agg-v2 1 VM 60 0.829 1.033 34327.5 +modify-global-flag-agg-v2 1 VM 60 0.954 1.021 4224.0 +modify-global-bounded-agg-v2 1 VM 60 0.869 1.076 7793.9 +modify-global-milestone-agg-v2 1 VM 60 0.879 1.022 25097.7 +resource-groups-global-write-tag1-kb 1 VM 60 0.921 1.025 9303.3 +resource-groups-global-write-and-read-tag1-kb 1 VM 60 0.955 1.020 5548.2 +resource-groups-sender-write-tag1-kb 1 VM 60 0.856 1.147 20568.6 +resource-groups-sender-multi-change1-kb 1 VM 60 0.905 1.099 17029.7 +token-v1ft-mint-and-transfer 1 VM 60 0.941 1.017 1162.9 +token-v1ft-mint-and-transfer 100 VM 60 0.880 1.021 18257.0 +token-v1nft-mint-and-transfer-sequential 1 VM 60 0.946 1.013 735.7 +token-v1nft-mint-and-transfer-sequential 100 VM 60 0.885 1.012 12888.4 +coin-init-and-mint 1 VM 60 0.803 1.024 27197.8 +coin-init-and-mint 100 VM 60 0.821 1.018 22715.7 +fungible-asset-mint 1 VM 60 0.820 1.024 23001.6 +fungible-asset-mint 100 VM 60 0.827 1.021 20084.2 +no-op5-signers 1 VM 60 0.821 1.024 38780.8 +token-v2-ambassador-mint 1 VM 60 0.818 1.019 15207.9 +token-v2-ambassador-mint 100 VM 60 0.871 1.021 15150.8 +liquidity-pool-swap 1 VM 60 0.928 1.023 814.2 +liquidity-pool-swap 100 VM 60 0.875 1.022 11378.2 +liquidity-pool-swap-stable 1 VM 60 0.930 1.028 781.5 +liquidity-pool-swap-stable 100 VM 60 0.903 1.019 11056.6 +deserialize-u256 1 VM 60 0.869 1.021 37002.8 +no-op-fee-payer 1 VM 60 0.952 1.026 2058.3 +no-op-fee-payer 100 VM 60 0.828 1.016 32284.2 +simple-script 1 VM 60 0.830 1.024 37707.7 +no_commit_apt-fa-transfer 1 VM 60 0.927 1.015 29357.5 +no_commit_apt-fa-transfer 1 NativeVM 60 0.922 1.021 44946.3 +no_commit_apt-fa-transfer 1 AptosVMSpeculative 60 0.953 1.010 1851.9 +no_commit_apt-fa-transfer 1 NativeSpeculative 60 0.789 1.031 109774.7