diff --git a/aptos-move/e2e-benchmark/data/calibration_values.tsv b/aptos-move/e2e-benchmark/data/calibration_values.tsv new file mode 100644 index 0000000000000..47ae8b7258196 --- /dev/null +++ b/aptos-move/e2e-benchmark/data/calibration_values.tsv @@ -0,0 +1,26 @@ +Loop { loop_count: Some(100000), loop_type: NoOp } 60 0.955 1.074 41893.7 +Loop { loop_count: Some(10000), loop_type: Arithmetic } 60 0.965 1.078 25915.0 +CreateObjects { num_objects: 10, object_payload_size: 0 } 60 0.924 1.082 158.1 +CreateObjects { num_objects: 10, object_payload_size: 10240 } 60 0.951 1.118 9356.2 +CreateObjects { num_objects: 100, object_payload_size: 0 } 60 0.926 1.082 1574.2 +CreateObjects { num_objects: 100, object_payload_size: 10240 } 60 0.952 1.092 11541.9 +InitializeVectorPicture { length: 128 } 10 0.965 1.038 163.3 +VectorPicture { length: 128 } 10 0.938 1.060 48.8 +VectorPictureRead { length: 128 } 10 0.977 1.077 46.4 +InitializeVectorPicture { length: 30720 } 60 0.948 1.123 27893.4 +VectorPicture { length: 30720 } 60 0.931 1.125 6923.1 +VectorPictureRead { length: 30720 } 60 0.934 1.102 6923.1 +SmartTablePicture { length: 30720, num_points_per_txn: 200 } 60 0.952 1.109 43594.7 +SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 60 0.957 1.120 73865.4 +ResourceGroupsSenderWriteTag { string_length: 1024 } 60 0.934 1.134 15.0 +ResourceGroupsSenderMultiChange { string_length: 1024 } 60 0.929 1.122 32.3 +TokenV1MintAndTransferFT 60 0.958 1.093 385.2 +TokenV1MintAndTransferNFTSequential 60 0.973 1.139 588.1 +TokenV2AmbassadorMint { numbered: true } 60 0.960 1.141 512.5 +LiquidityPoolSwap { is_stable: true } 60 0.961 1.103 590.3 +LiquidityPoolSwap { is_stable: false } 60 0.954 1.134 552.2 +CoinInitAndMint 10 0.975 1.043 199.6 +FungibleAssetMint 10 0.954 1.038 236.3 +IncGlobalMilestoneAggV2 { milestone_every: 1 } 10 0.960 1.047 32.9 +IncGlobalMilestoneAggV2 { milestone_every: 2 } 10 0.971 1.066 18.1 +EmitEvents { count: 1000 } 10 0.969 1.052 8615.5 \ No newline at end of file diff --git a/aptos-move/e2e-benchmark/src/main.rs b/aptos-move/e2e-benchmark/src/main.rs index e00db94985007..252bf800bd79e 100644 --- a/aptos-move/e2e-benchmark/src/main.rs +++ b/aptos-move/e2e-benchmark/src/main.rs @@ -15,7 +15,7 @@ use aptos_transaction_generator_lib::{ use aptos_types::{account_address::AccountAddress, transaction::TransactionPayload}; use rand::{rngs::StdRng, SeedableRng}; use serde_json::json; -use std::{collections::HashMap, process::exit}; +use std::{collections::HashMap, fs, process::exit}; // bump after a bigger test or perf change, so you can easily distinguish runs // that are on top of this commit @@ -85,42 +85,14 @@ const ALLOWED_REGRESSION: f64 = 0.15; const ALLOWED_IMPROVEMENT: f64 = 0.15; const ABSOLUTE_BUFFER_US: f64 = 2.0; -const CALIBRATION_VALUES: &str = " -Loop { loop_count: Some(100000), loop_type: NoOp } 60 0.955 1.074 41893.7 -Loop { loop_count: Some(10000), loop_type: Arithmetic } 60 0.965 1.078 25915.0 -CreateObjects { num_objects: 10, object_payload_size: 0 } 60 0.924 1.082 158.1 -CreateObjects { num_objects: 10, object_payload_size: 10240 } 60 0.951 1.118 9356.2 -CreateObjects { num_objects: 100, object_payload_size: 0 } 60 0.926 1.082 1574.2 -CreateObjects { num_objects: 100, object_payload_size: 10240 } 60 0.952 1.092 11541.9 -InitializeVectorPicture { length: 128 } 10 0.965 1.038 163.3 -VectorPicture { length: 128 } 10 0.938 1.060 48.8 -VectorPictureRead { length: 128 } 10 0.977 1.077 46.4 -InitializeVectorPicture { length: 30720 } 60 0.948 1.123 27893.4 -VectorPicture { length: 30720 } 60 0.931 1.125 6923.1 -VectorPictureRead { length: 30720 } 60 0.934 1.102 6923.1 -SmartTablePicture { length: 30720, num_points_per_txn: 200 } 60 0.952 1.109 43594.7 -SmartTablePicture { length: 1048576, num_points_per_txn: 300 } 60 0.957 1.120 73865.4 -ResourceGroupsSenderWriteTag { string_length: 1024 } 60 0.934 1.134 15.0 -ResourceGroupsSenderMultiChange { string_length: 1024 } 60 0.929 1.122 32.3 -TokenV1MintAndTransferFT 60 0.958 1.093 385.2 -TokenV1MintAndTransferNFTSequential 60 0.973 1.139 588.1 -TokenV2AmbassadorMint { numbered: true } 60 0.960 1.141 512.5 -LiquidityPoolSwap { is_stable: true } 60 0.961 1.103 590.3 -LiquidityPoolSwap { is_stable: false } 60 0.954 1.134 552.2 -CoinInitAndMint 10 0.975 1.043 199.6 -FungibleAssetMint 10 0.954 1.038 236.3 -IncGlobalMilestoneAggV2 { milestone_every: 1 } 10 0.960 1.047 32.9 -IncGlobalMilestoneAggV2 { milestone_every: 2 } 10 0.971 1.066 18.1 -EmitEvents { count: 1000 } 10 0.969 1.052 8615.5 -"; - struct CalibrationInfo { // count: usize, expected_time_micros: f64, } fn get_parsed_calibration_values() -> HashMap { - CALIBRATION_VALUES + let calibration_values = fs::read_to_string("aptos-move/e2e-benchmark/data/calibration_values.tsv").expect("Unable to read file"); + calibration_values .trim() .split('\n') .map(|line| { diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 8d78dc89d8b9d..171ff766a2a67 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -166,52 +166,9 @@ class RunGroupConfig: CALIBRATION_SEPARATOR = " " # transaction_type module_working_set_size executor_type count min_ratio max_ratio median -CALIBRATION = """ -no-op 1 VM 6 0.938 1.019 38925.3 -no-op 1000 VM 6 0.943 1.019 36444.6 -apt-fa-transfer 1 VM 6 0.927 1.018 26954.7 -apt-fa-transfer 1 NativeVM 6 0.927 1.018 35259.7 -account-generation 1 VM 6 0.96 1.02 20606.2 -account-generation 1 NativeVM 6 0.96 1.02 28216.2 -account-resource32-b 1 VM 6 0.94 1.026 34260.4 -modify-global-resource 1 VM 6 0.993 1.021 2260.5 -modify-global-resource 100 VM 6 0.982 1.02 33129.7 -publish-package 1 VM 6 0.983 1.012 1672.6 -mix_publish_transfer 1 VM 6 0.972 1.044 20832.8 -batch100-transfer 1 VM 6 0.953 1.024 645.1 -batch100-transfer 1 NativeVM 6 0.953 1.024 1437.0 -vector-picture30k 1 VM 6 0.992 1.039 103.6 -vector-picture30k 100 VM 6 0.913 1.015 1831.5 -smart-table-picture30-k-with200-change 1 VM 6 0.976 1.034 16.1 -smart-table-picture30-k-with200-change 100 VM 6 0.985 1.018 212.9 -modify-global-resource-agg-v2 1 VM 6 0.976 1.035 33992.5 -modify-global-flag-agg-v2 1 VM 6 0.986 1.016 4224 -modify-global-bounded-agg-v2 1 VM 6 0.964 1.047 7661.6 -modify-global-milestone-agg-v2 1 VM 6 0.973 1.017 25187.1 -resource-groups-global-write-tag1-kb 1 VM 6 0.989 1.03 9215.7 -resource-groups-global-write-and-read-tag1-kb 1 VM 6 0.982 1.018 5538.3 -resource-groups-sender-write-tag1-kb 1 VM 6 0.985 1.059 20084.2 -resource-groups-sender-multi-change1-kb 1 VM 6 0.968 1.034 16400.4 -token-v1ft-mint-and-transfer 1 VM 6 0.987 1.022 1156.3 -token-v1ft-mint-and-transfer 100 VM 6 0.964 1.024 17842.6 -token-v1nft-mint-and-transfer-sequential 1 VM 6 0.984 1.017 735.7 -token-v1nft-mint-and-transfer-sequential 100 VM 6 0.966 1.017 12819.7 -coin-init-and-mint 1 VM 6 0.95 1.024 26906.4 -coin-init-and-mint 100 VM 6 0.985 1.022 22312.6 -fungible-asset-mint 1 VM 6 0.955 1.013 23001.6 -fungible-asset-mint 100 VM 6 0.955 1.015 19973.5 -no-op5-signers 1 VM 6 0.934 1.016 38708.6 -token-v2-ambassador-mint 1 VM 6 0.975 1.008 15179.3 -token-v2-ambassador-mint 100 VM 6 0.985 1.007 15150.8 -liquidity-pool-swap 1 VM 6 0.987 1.018 805.5 -liquidity-pool-swap 100 VM 6 0.993 1.02 11156.3 -liquidity-pool-swap-stable 1 VM 6 0.985 1.017 778.7 -liquidity-pool-swap-stable 100 VM 6 0.982 1.009 11056.6 -deserialize-u256 1 VM 6 0.968 1.026 36444.6 -no-op-fee-payer 1 VM 6 0.994 1.026 2046 -no-op-fee-payer 100 VM 6 0.96 1.014 32866.5 -simple-script 1 VM 6 0.941 1.012 38206.1 -""" +with open('testsuite/single_node_performance_values.tsv', 'r') as file: + CALIBRATION = file.read() + # when adding a new test, add estimated expected_tps to it, as well as waived=True. # And then after a day or two - add calibration result for it above, removing expected_tps/waived fields. diff --git a/testsuite/single_node_performance_calibration.py b/testsuite/single_node_performance_calibration.py new file mode 100755 index 0000000000000..e33d60e427483 --- /dev/null +++ b/testsuite/single_node_performance_calibration.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python + +import argparse +import requests + +def humio_secret(): + import subprocess + return subprocess.run(["gcloud", "secrets", "versions", "access", "--secret=ci_humio_read_token", "--project=aptos-shared-secrets", "latest"], capture_output=True).stdout.decode("utf-8") + + +def humio_url(): + return "https://cloud.us.humio.com/api/v1/repositories/github/query" + +def parse_args(): + parser = argparse.ArgumentParser(description='Benchmark calibration tools') + + parser.add_argument( + '--branch', + type=str, + help='Optional branch, if passed - only looks at results run on that branch through adhoc runs', + ) + + parser.add_argument( + '--move-e2e', + action='store_true', + help='Calibrate move e2e test', + ) + + return parser.parse_args() + +def query_humio(query_string, time_interval="5d"): + query = { + "queryString": query_string, + "start": time_interval, + } + + secret = humio_secret() + + resp = requests.post( + url=humio_url(), + json=query, + headers={ + "Authorization": f"Bearer {secret}", + "Content-Type": "application/json", + }, + ) + + return resp.text.strip() + +def main(): + args = parse_args() + + if args.move_e2e: + prefix = ( + """ + github.job.name = "single-node-performance" + | github.workflow.head_branch = "{branch}" + | "grep_json_aptos_move_vm_perf" + | parseJson(message) + """.format( + branch=args.branch + ) + if args.branch is not None + else """ + github.job.name = "execution-performance / single-node-performance" + | github.workflow.head_branch = "main" + | "grep_json_aptos_move_vm_perf" + | parseJson(message) + """ + ) + + query_string = ( + prefix + + """ + | groupBy([test_index, transaction_type], function=[count(as="count"), avg(expected_wall_time_us, as="expected"), avg(wall_time_us, as="avg_wall_time_us"), min(wall_time_us, as="min_wall_time_us"), max(wall_time_us, as="max_wall_time_us"), percentile(field=wall_time_us, accuracy=0.001, percentiles=[50])]) + | min_ratio := min_wall_time_us / _50 + | avg_ratio := avg_wall_time_us / _50 + | max_ratio := max_wall_time_us / _50 + | offset_avg_from_expected := _50 / expected + | format("%.1f", field=_50, as="median") + | format("%.1f", field=avg_wall_time_us, as="avg_wall_time_us") + | format("%.1f", field=min_wall_time_us, as="min_wall_time_us") + | format("%.1f", field=max_wall_time_us, as="max_wall_time_us") + | format("%.3f", field=min_ratio, as="min_ratio") + | format("%.3f", field=max_ratio, as="max_ratio") + | format("%.3f", field=offset_avg_from_expected, as="offset_median_from_expected") + | table([transaction_type, count, min_ratio, max_ratio, median, expected], sortby=test_index, reverse=false) + """ + ) + + columns = ["transaction_type", "count", "min_ratio", "max_ratio", "median"] + + def split_line(line): + line = line.strip() + if "}" in line: + parts = line.split("}") + res = [parts[0] + "}"] + parts[1].split(", ")[1:] + return res + else: + return line.split(", ") + output_file_name = "aptos-move/e2e-benchmark/data/calibration_values.tsv" + + else: + prefix = ( + """ + github.job.name = "single-node-performance" + | github.workflow.head_branch = "{branch}" + | "grep_json_single_node_perf" + | parseJson(message) + | source = "ADHOC" + | code_perf_version = "v8" + """.format( + branch=args.branch + ) + if args.branch is not None + else """ + github.job.name = "execution-performance / single-node-performance" + | github.workflow.head_branch = "main" + | "grep_json_single_node_perf" + | parseJson(message) + """ + ) + + query_string = ( + prefix + + """ + | groupBy([test_index, transaction_type, module_working_set_size, executor_type, code_perf_version], function=[count(as="count"), avg(expected_tps, as="expected"), avg(tps, as="avg_tps"), min(tps, as="min_tps"), max(tps, as="max_tps"), percentile(field=tps, accuracy=0.001, percentiles=[50])]) + | min_ratio := min_tps / _50 + | avg_ratio := avg_tps / _50 + | max_ratio := max_tps / _50 + | offset_avg_from_expected := _50 / expected + | format("%.1f", field=_50, as="median") + | format("%.1f", field=avg_tps, as="avg_tps") + | format("%.1f", field=min_tps, as="min_tps") + | format("%.1f", field=max_tps, as="max_tps") + | format("%.3f", field=min_ratio, as="min_ratio") + | format("%.3f", field=max_ratio, as="max_ratio") + | format("%.3f", field=offset_avg_from_expected, as="offset_median_from_expected") + | table([transaction_type, module_working_set_size, executor_type, count, min_ratio, max_ratio, median], sortby=test_index, reverse=false) + """ + ) + + columns = [ + "transaction_type", + "module_working_set_size", + "executor_type", + "count", + "min_ratio", + "max_ratio", + "median", + ] + + def split_line(line): + return line.strip().split(", ") + + output_file_name = "testsuite/single_node_performance_values.tsv" + + response_text = query_humio(query_string) + + parsed = [ + {(parts := key_value.split("->"))[0]: parts[1] for key_value in split_line(line)} + for line in response_text.split("\n") + ] + + with open(output_file_name, "w") as f: + for line in parsed: + f.write("\t".join([line[column] for column in columns])) + f.write("\n") + + print(f"Written to {output_file_name}") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/testsuite/single_node_performance_values.tsv b/testsuite/single_node_performance_values.tsv new file mode 100644 index 0000000000000..57d7be160aca6 --- /dev/null +++ b/testsuite/single_node_performance_values.tsv @@ -0,0 +1,44 @@ +no-op 1 VM 6 0.938 1.019 38925.3 +no-op 1000 VM 6 0.943 1.019 36444.6 +apt-fa-transfer 1 VM 6 0.927 1.018 26954.7 +apt-fa-transfer 1 NativeVM 6 0.927 1.018 35259.7 +account-generation 1 VM 6 0.96 1.02 20606.2 +account-generation 1 NativeVM 6 0.96 1.02 28216.2 +account-resource32-b 1 VM 6 0.94 1.026 34260.4 +modify-global-resource 1 VM 6 0.993 1.021 2260.5 +modify-global-resource 100 VM 6 0.982 1.02 33129.7 +publish-package 1 VM 6 0.983 1.012 1672.6 +mix_publish_transfer 1 VM 6 0.972 1.044 20832.8 +batch100-transfer 1 VM 6 0.953 1.024 645.1 +batch100-transfer 1 NativeVM 6 0.953 1.024 1437.0 +vector-picture30k 1 VM 6 0.992 1.039 103.6 +vector-picture30k 100 VM 6 0.913 1.015 1831.5 +smart-table-picture30-k-with200-change 1 VM 6 0.976 1.034 16.1 +smart-table-picture30-k-with200-change 100 VM 6 0.985 1.018 212.9 +modify-global-resource-agg-v2 1 VM 6 0.976 1.035 33992.5 +modify-global-flag-agg-v2 1 VM 6 0.986 1.016 4224 +modify-global-bounded-agg-v2 1 VM 6 0.964 1.047 7661.6 +modify-global-milestone-agg-v2 1 VM 6 0.973 1.017 25187.1 +resource-groups-global-write-tag1-kb 1 VM 6 0.989 1.03 9215.7 +resource-groups-global-write-and-read-tag1-kb 1 VM 6 0.982 1.018 5538.3 +resource-groups-sender-write-tag1-kb 1 VM 6 0.985 1.059 20084.2 +resource-groups-sender-multi-change1-kb 1 VM 6 0.968 1.034 16400.4 +token-v1ft-mint-and-transfer 1 VM 6 0.987 1.022 1156.3 +token-v1ft-mint-and-transfer 100 VM 6 0.964 1.024 17842.6 +token-v1nft-mint-and-transfer-sequential 1 VM 6 0.984 1.017 735.7 +token-v1nft-mint-and-transfer-sequential 100 VM 6 0.966 1.017 12819.7 +coin-init-and-mint 1 VM 6 0.95 1.024 26906.4 +coin-init-and-mint 100 VM 6 0.985 1.022 22312.6 +fungible-asset-mint 1 VM 6 0.955 1.013 23001.6 +fungible-asset-mint 100 VM 6 0.955 1.015 19973.5 +no-op5-signers 1 VM 6 0.934 1.016 38708.6 +token-v2-ambassador-mint 1 VM 6 0.975 1.008 15179.3 +token-v2-ambassador-mint 100 VM 6 0.985 1.007 15150.8 +liquidity-pool-swap 1 VM 6 0.987 1.018 805.5 +liquidity-pool-swap 100 VM 6 0.993 1.02 11156.3 +liquidity-pool-swap-stable 1 VM 6 0.985 1.017 778.7 +liquidity-pool-swap-stable 100 VM 6 0.982 1.009 11056.6 +deserialize-u256 1 VM 6 0.968 1.026 36444.6 +no-op-fee-payer 1 VM 6 0.994 1.026 2046 +no-op-fee-payer 100 VM 6 0.96 1.014 32866.5 +simple-script 1 VM 6 0.941 1.012 38206.1 \ No newline at end of file