From 9a2e48fcfb1328e630cadd16c175be063c589aa1 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Mon, 17 Jun 2024 16:21:25 -0700 Subject: [PATCH] add perf improvment to unblock --- Cargo.lock | 15 --------------- api/Cargo.toml | 1 - .../indexer-grpc/indexer-grpc-fullnode/Cargo.toml | 1 - storage/indexer/Cargo.toml | 10 +--------- storage/indexer/src/db_indexer.rs | 7 +++---- storage/indexer_schemas/Cargo.toml | 7 ------- testsuite/single_node_performance.py | 4 ++-- 7 files changed, 6 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37e7dd7cb9ab37..b8b6d3e0197054 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,7 +401,6 @@ dependencies = [ "aptos-cached-packages", "aptos-config", "aptos-crypto", - "aptos-db-indexer", "aptos-framework", "aptos-gas-meter", "aptos-gas-schedule", @@ -1166,19 +1165,13 @@ dependencies = [ "aptos-resource-viewer", "aptos-rocksdb-options", "aptos-schemadb", - "aptos-sdk", "aptos-storage-interface", "aptos-types", - "aptos-vm-genesis", "bcs 0.1.4", - "byteorder", "bytes", "dashmap", "move-core-types", - "proptest", - "proptest-derive", "rand 0.7.3", - "serde", ] [[package]] @@ -1186,19 +1179,12 @@ name = "aptos-db-indexer-schemas" version = "0.1.0" dependencies = [ "anyhow", - "aptos-config", - "aptos-logger", "aptos-proptest-helpers", - "aptos-resource-viewer", - "aptos-rocksdb-options", "aptos-schemadb", "aptos-storage-interface", "aptos-types", "bcs 0.1.4", "byteorder", - "bytes", - "dashmap", - "move-core-types", "proptest", "proptest-derive", "rand 0.7.3", @@ -2072,7 +2058,6 @@ dependencies = [ "aptos-config", "aptos-crypto", "aptos-db", - "aptos-db-indexer", "aptos-executor", "aptos-executor-types", "aptos-framework", diff --git a/api/Cargo.toml b/api/Cargo.toml index 2aac368f028e29..c037526d47d020 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -19,7 +19,6 @@ aptos-bcs-utils = { workspace = true } aptos-build-info = { workspace = true } aptos-config = { workspace = true } aptos-crypto = { workspace = true } -aptos-db-indexer = { workspace = true } aptos-gas-schedule = { workspace = true } aptos-global-constants = { workspace = true } aptos-logger = { workspace = true } diff --git a/ecosystem/indexer-grpc/indexer-grpc-fullnode/Cargo.toml b/ecosystem/indexer-grpc/indexer-grpc-fullnode/Cargo.toml index 15aacecc240946..99587b126974d7 100644 --- a/ecosystem/indexer-grpc/indexer-grpc-fullnode/Cargo.toml +++ b/ecosystem/indexer-grpc/indexer-grpc-fullnode/Cargo.toml @@ -33,7 +33,6 @@ aptos-api = { workspace = true } aptos-api-types = { workspace = true } aptos-bitvec = { workspace = true } aptos-config = { workspace = true } -aptos-db-indexer = { workspace = true } aptos-indexer-grpc-utils = { workspace = true } aptos-logger = { workspace = true } aptos-mempool = { workspace = true } diff --git a/storage/indexer/Cargo.toml b/storage/indexer/Cargo.toml index 58bba0559e724a..963c1a61864b0c 100644 --- a/storage/indexer/Cargo.toml +++ b/storage/indexer/Cargo.toml @@ -20,27 +20,19 @@ aptos-logger = { workspace = true } aptos-resource-viewer = { workspace = true } aptos-rocksdb-options = { workspace = true } aptos-schemadb = { workspace = true } -aptos-sdk = { workspace = true } aptos-storage-interface = { workspace = true } aptos-types = { workspace = true } -aptos-vm-genesis = { workspace = true } bcs = { workspace = true } -byteorder = { workspace = true } bytes = { workspace = true } dashmap = { workspace = true } move-core-types = { workspace = true } -proptest = { workspace = true, optional = true } -proptest-derive = { workspace = true, optional = true } -serde = { workspace = true } [dev-dependencies] aptos-proptest-helpers = { workspace = true } aptos-schemadb = { workspace = true, features = ["fuzzing"] } aptos-types = { workspace = true, features = ["fuzzing"] } -proptest = { workspace = true } -proptest-derive = { workspace = true } rand = { workspace = true } [features] default = [] -fuzzing = ["proptest", "proptest-derive", "aptos-types/fuzzing", "aptos-schemadb/fuzzing"] +fuzzing = ["aptos-types/fuzzing", "aptos-schemadb/fuzzing"] diff --git a/storage/indexer/src/db_indexer.rs b/storage/indexer/src/db_indexer.rs index b01e1c906fddd8..7b0dd635356c07 100644 --- a/storage/indexer/src/db_indexer.rs +++ b/storage/indexer/src/db_indexer.rs @@ -143,10 +143,9 @@ impl DBIndexer { fn get_num_of_transactions(&self, version: Version) -> Result { let highest_version = self.main_db_reader.get_synced_version()?; - Ok(min( - self.config.batch_size as u64, - highest_version - version, - )) + // we want to include the last transaction since the iterator interface will is right exclusive. + let num_of_transaction = min(self.config.batch_size as u64, highest_version - version) + 1; + Ok(num_of_transaction) } pub fn process_a_batch(&self, start_version: Option) -> Result { diff --git a/storage/indexer_schemas/Cargo.toml b/storage/indexer_schemas/Cargo.toml index 8c205737892740..1768053160c89c 100644 --- a/storage/indexer_schemas/Cargo.toml +++ b/storage/indexer_schemas/Cargo.toml @@ -14,18 +14,11 @@ rust-version = { workspace = true } [dependencies] anyhow = { workspace = true } -aptos-config = { workspace = true } -aptos-logger = { workspace = true } -aptos-resource-viewer = { workspace = true } -aptos-rocksdb-options = { workspace = true } aptos-schemadb = { workspace = true } aptos-storage-interface = { workspace = true } aptos-types = { workspace = true } bcs = { workspace = true } byteorder = { workspace = true } -bytes = { workspace = true } -dashmap = { workspace = true } -move-core-types = { workspace = true } proptest = { workspace = true, optional = true } proptest-derive = { workspace = true, optional = true } serde = { workspace = true } diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 85335eae3e7d89..eac99808114599 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -89,7 +89,7 @@ class RunGroupConfig: RunGroupConfig(expected_tps=21300, key=RunGroupKey("no-op"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=11500, key=RunGroupKey("no-op", module_working_set_size=1000), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=12800, key=RunGroupKey("coin-transfer"), included_in=LAND_BLOCKING_AND_C | Flow.REPRESENTATIVE), - RunGroupConfig(expected_tps=32817, key=RunGroupKey("coin-transfer", executor_type="native"), included_in=LAND_BLOCKING_AND_C), + RunGroupConfig(expected_tps=34500, key=RunGroupKey("coin-transfer", executor_type="native"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=9000, key=RunGroupKey("account-generation"), included_in=LAND_BLOCKING_AND_C | Flow.REPRESENTATIVE), RunGroupConfig(expected_tps=27873, key=RunGroupKey("account-generation", executor_type="native"), included_in=Flow.CONTINUOUS), RunGroupConfig(expected_tps=18600, key=RunGroupKey("account-resource32-b"), included_in=Flow.CONTINUOUS), @@ -121,7 +121,7 @@ class RunGroupConfig: RunGroupConfig(expected_tps=12500, key=RunGroupKey("modify-global-bounded-agg-v2", module_working_set_size=50), included_in=Flow.AGG_V2), RunGroupConfig(expected_tps=16195, key=RunGroupKey("modify-global-milestone-agg-v2"), included_in=Flow.AGG_V2 | Flow.CONTINUOUS), - RunGroupConfig(expected_tps=7900, key=RunGroupKey("resource-groups-global-write-tag1-kb"), included_in=LAND_BLOCKING_AND_C | Flow.RESOURCE_GROUPS), + RunGroupConfig(expected_tps=7920, key=RunGroupKey("resource-groups-global-write-tag1-kb"), included_in=LAND_BLOCKING_AND_C | Flow.RESOURCE_GROUPS), RunGroupConfig(expected_tps=8000, key=RunGroupKey("resource-groups-global-write-tag1-kb", module_working_set_size=20), included_in=Flow.RESOURCE_GROUPS, waived=True), RunGroupConfig(expected_tps=6000, key=RunGroupKey("resource-groups-global-write-and-read-tag1-kb"), included_in=Flow.CONTINUOUS | Flow.RESOURCE_GROUPS), RunGroupConfig(expected_tps=8000, key=RunGroupKey("resource-groups-global-write-and-read-tag1-kb", module_working_set_size=20), included_in=Flow.RESOURCE_GROUPS, waived=True),