From d072cd4d087895794c72ac50e2c79f4055a29b71 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Mon, 17 Jun 2024 16:21:25 -0700 Subject: [PATCH] add perf improvment to unblock --- Cargo.lock | 15 --------------- api/Cargo.toml | 1 - .../indexer-grpc-fullnode/Cargo.toml | 1 - storage/indexer/Cargo.toml | 10 +--------- storage/indexer/src/db_indexer.rs | 7 +++---- storage/indexer_schemas/Cargo.toml | 7 ------- testsuite/single_node_performance.py | 4 ++-- .../std-lib-conflicts/Move.compile | 0 .../compilation/std-lib-conflicts/Move.exp | 1 - .../compilation/std-lib-conflicts/Move.toml | 19 ------------------- .../std-lib-conflicts/deps_only/Move.toml | 16 ---------------- .../deps_only/sources/B.move | 3 --- .../std-lib-conflicts/sources/A.move | 6 ------ 13 files changed, 6 insertions(+), 84 deletions(-) delete mode 100644 third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.compile delete mode 100644 third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.exp delete mode 100644 third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.toml delete mode 100644 third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/deps_only/Move.toml delete mode 100644 third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/deps_only/sources/B.move delete mode 100644 third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/sources/A.move diff --git a/Cargo.lock b/Cargo.lock index 37e7dd7cb9ab37..b8b6d3e0197054 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,7 +401,6 @@ dependencies = [ "aptos-cached-packages", "aptos-config", "aptos-crypto", - "aptos-db-indexer", "aptos-framework", "aptos-gas-meter", "aptos-gas-schedule", @@ -1166,19 +1165,13 @@ dependencies = [ "aptos-resource-viewer", "aptos-rocksdb-options", "aptos-schemadb", - "aptos-sdk", "aptos-storage-interface", "aptos-types", - "aptos-vm-genesis", "bcs 0.1.4", - "byteorder", "bytes", "dashmap", "move-core-types", - "proptest", - "proptest-derive", "rand 0.7.3", - "serde", ] [[package]] @@ -1186,19 +1179,12 @@ name = "aptos-db-indexer-schemas" version = "0.1.0" dependencies = [ "anyhow", - "aptos-config", - "aptos-logger", "aptos-proptest-helpers", - "aptos-resource-viewer", - "aptos-rocksdb-options", "aptos-schemadb", "aptos-storage-interface", "aptos-types", "bcs 0.1.4", "byteorder", - "bytes", - "dashmap", - "move-core-types", "proptest", "proptest-derive", "rand 0.7.3", @@ -2072,7 +2058,6 @@ dependencies = [ "aptos-config", "aptos-crypto", "aptos-db", - "aptos-db-indexer", "aptos-executor", "aptos-executor-types", "aptos-framework", diff --git a/api/Cargo.toml b/api/Cargo.toml index 2aac368f028e29..c037526d47d020 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -19,7 +19,6 @@ aptos-bcs-utils = { workspace = true } aptos-build-info = { workspace = true } aptos-config = { workspace = true } aptos-crypto = { workspace = true } -aptos-db-indexer = { workspace = true } aptos-gas-schedule = { workspace = true } aptos-global-constants = { workspace = true } aptos-logger = { workspace = true } diff --git a/ecosystem/indexer-grpc/indexer-grpc-fullnode/Cargo.toml b/ecosystem/indexer-grpc/indexer-grpc-fullnode/Cargo.toml index 15aacecc240946..99587b126974d7 100644 --- a/ecosystem/indexer-grpc/indexer-grpc-fullnode/Cargo.toml +++ b/ecosystem/indexer-grpc/indexer-grpc-fullnode/Cargo.toml @@ -33,7 +33,6 @@ aptos-api = { workspace = true } aptos-api-types = { workspace = true } aptos-bitvec = { workspace = true } aptos-config = { workspace = true } -aptos-db-indexer = { workspace = true } aptos-indexer-grpc-utils = { workspace = true } aptos-logger = { workspace = true } aptos-mempool = { workspace = true } diff --git a/storage/indexer/Cargo.toml b/storage/indexer/Cargo.toml index 58bba0559e724a..963c1a61864b0c 100644 --- a/storage/indexer/Cargo.toml +++ b/storage/indexer/Cargo.toml @@ -20,27 +20,19 @@ aptos-logger = { workspace = true } aptos-resource-viewer = { workspace = true } aptos-rocksdb-options = { workspace = true } aptos-schemadb = { workspace = true } -aptos-sdk = { workspace = true } aptos-storage-interface = { workspace = true } aptos-types = { workspace = true } -aptos-vm-genesis = { workspace = true } bcs = { workspace = true } -byteorder = { workspace = true } bytes = { workspace = true } dashmap = { workspace = true } move-core-types = { workspace = true } -proptest = { workspace = true, optional = true } -proptest-derive = { workspace = true, optional = true } -serde = { workspace = true } [dev-dependencies] aptos-proptest-helpers = { workspace = true } aptos-schemadb = { workspace = true, features = ["fuzzing"] } aptos-types = { workspace = true, features = ["fuzzing"] } -proptest = { workspace = true } -proptest-derive = { workspace = true } rand = { workspace = true } [features] default = [] -fuzzing = ["proptest", "proptest-derive", "aptos-types/fuzzing", "aptos-schemadb/fuzzing"] +fuzzing = ["aptos-types/fuzzing", "aptos-schemadb/fuzzing"] diff --git a/storage/indexer/src/db_indexer.rs b/storage/indexer/src/db_indexer.rs index b01e1c906fddd8..7b0dd635356c07 100644 --- a/storage/indexer/src/db_indexer.rs +++ b/storage/indexer/src/db_indexer.rs @@ -143,10 +143,9 @@ impl DBIndexer { fn get_num_of_transactions(&self, version: Version) -> Result { let highest_version = self.main_db_reader.get_synced_version()?; - Ok(min( - self.config.batch_size as u64, - highest_version - version, - )) + // we want to include the last transaction since the iterator interface will is right exclusive. + let num_of_transaction = min(self.config.batch_size as u64, highest_version - version) + 1; + Ok(num_of_transaction) } pub fn process_a_batch(&self, start_version: Option) -> Result { diff --git a/storage/indexer_schemas/Cargo.toml b/storage/indexer_schemas/Cargo.toml index 8c205737892740..1768053160c89c 100644 --- a/storage/indexer_schemas/Cargo.toml +++ b/storage/indexer_schemas/Cargo.toml @@ -14,18 +14,11 @@ rust-version = { workspace = true } [dependencies] anyhow = { workspace = true } -aptos-config = { workspace = true } -aptos-logger = { workspace = true } -aptos-resource-viewer = { workspace = true } -aptos-rocksdb-options = { workspace = true } aptos-schemadb = { workspace = true } aptos-storage-interface = { workspace = true } aptos-types = { workspace = true } bcs = { workspace = true } byteorder = { workspace = true } -bytes = { workspace = true } -dashmap = { workspace = true } -move-core-types = { workspace = true } proptest = { workspace = true, optional = true } proptest-derive = { workspace = true, optional = true } serde = { workspace = true } diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 85335eae3e7d89..eac99808114599 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -89,7 +89,7 @@ class RunGroupConfig: RunGroupConfig(expected_tps=21300, key=RunGroupKey("no-op"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=11500, key=RunGroupKey("no-op", module_working_set_size=1000), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=12800, key=RunGroupKey("coin-transfer"), included_in=LAND_BLOCKING_AND_C | Flow.REPRESENTATIVE), - RunGroupConfig(expected_tps=32817, key=RunGroupKey("coin-transfer", executor_type="native"), included_in=LAND_BLOCKING_AND_C), + RunGroupConfig(expected_tps=34500, key=RunGroupKey("coin-transfer", executor_type="native"), included_in=LAND_BLOCKING_AND_C), RunGroupConfig(expected_tps=9000, key=RunGroupKey("account-generation"), included_in=LAND_BLOCKING_AND_C | Flow.REPRESENTATIVE), RunGroupConfig(expected_tps=27873, key=RunGroupKey("account-generation", executor_type="native"), included_in=Flow.CONTINUOUS), RunGroupConfig(expected_tps=18600, key=RunGroupKey("account-resource32-b"), included_in=Flow.CONTINUOUS), @@ -121,7 +121,7 @@ class RunGroupConfig: RunGroupConfig(expected_tps=12500, key=RunGroupKey("modify-global-bounded-agg-v2", module_working_set_size=50), included_in=Flow.AGG_V2), RunGroupConfig(expected_tps=16195, key=RunGroupKey("modify-global-milestone-agg-v2"), included_in=Flow.AGG_V2 | Flow.CONTINUOUS), - RunGroupConfig(expected_tps=7900, key=RunGroupKey("resource-groups-global-write-tag1-kb"), included_in=LAND_BLOCKING_AND_C | Flow.RESOURCE_GROUPS), + RunGroupConfig(expected_tps=7920, key=RunGroupKey("resource-groups-global-write-tag1-kb"), included_in=LAND_BLOCKING_AND_C | Flow.RESOURCE_GROUPS), RunGroupConfig(expected_tps=8000, key=RunGroupKey("resource-groups-global-write-tag1-kb", module_working_set_size=20), included_in=Flow.RESOURCE_GROUPS, waived=True), RunGroupConfig(expected_tps=6000, key=RunGroupKey("resource-groups-global-write-and-read-tag1-kb"), included_in=Flow.CONTINUOUS | Flow.RESOURCE_GROUPS), RunGroupConfig(expected_tps=8000, key=RunGroupKey("resource-groups-global-write-and-read-tag1-kb", module_working_set_size=20), included_in=Flow.RESOURCE_GROUPS, waived=True), diff --git a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.compile b/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.compile deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.exp b/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.exp deleted file mode 100644 index d3c93b38103764..00000000000000 --- a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.exp +++ /dev/null @@ -1 +0,0 @@ -Unable to resolve packages for package 'std-lib-conflicts': While resolving dependency 'B' in package 'std-lib-conflicts': Unable to resolve package dependency 'B': While resolving dependency 'AptosFramework' in package 'B': Unable to resolve package dependency 'AptosFramework': Conflicting dependencies found: package 'AptosFramework' conflicts with 'AptosFramework' diff --git a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.toml b/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.toml deleted file mode 100644 index b6220b8e4c7b00..00000000000000 --- a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/Move.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "std-lib-conflicts" -version = "1.0.0" -authors = [] - -[addresses] -A = "0x42" - -[dev-addresses] - -[dependencies.AptosFramework] -git = "https://github.com/aptos-labs/aptos-core.git" -rev = "devnet" -subdir = "aptos-move/framework/aptos-framework" - -[dependencies.B] -local = "./deps_only" - -[dev-dependencies] diff --git a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/deps_only/Move.toml b/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/deps_only/Move.toml deleted file mode 100644 index c056666dea12d5..00000000000000 --- a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/deps_only/Move.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "B" -version = "1.0.0" -authors = [] - -[addresses] -B = "0x43" - -[dev-addresses] - -[dependencies.AptosFramework] -git = "https://github.com/aptos-labs/aptos-core.git" -rev = "mainnet" -subdir = "aptos-move/framework/aptos-framework" - -[dev-dependencies] diff --git a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/deps_only/sources/B.move b/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/deps_only/sources/B.move deleted file mode 100644 index 6167a02cb4ac2c..00000000000000 --- a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/deps_only/sources/B.move +++ /dev/null @@ -1,3 +0,0 @@ -module B::B { - public fun foo() {} -} diff --git a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/sources/A.move b/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/sources/A.move deleted file mode 100644 index 52551bda99f850..00000000000000 --- a/third_party/move/tools/move-package/tests/test_sources/compilation/std-lib-conflicts/sources/A.move +++ /dev/null @@ -1,6 +0,0 @@ -module A::A { - use B::B; - public fun foo() { - B::foo(); - } -}