From a9ea38382efd0b91f860da120b18d98e55acf72b Mon Sep 17 00:00:00 2001 From: Dev Ojha Date: Wed, 10 Nov 2021 10:20:37 -0600 Subject: [PATCH] perf: store/cachekv: Correct the naming and implementation for existing benchmarks (#10116) ## Description Cref discussion in #10026, updates the CacheKV Benchmark naming and implementation to correspond to whats actually going on, and remove many irrelevant/incorrect components from being in the benchmarks timing. Basically the old Benchmark's iterator creation was very flawed, and never hit the complex case, only repeatedly performing the best case performance of the iterator. Instead it really benchmarked iterator set and next operations. This PR splits out the benchmarks for those two accordingly. --- ### Author Checklist *All items are required. Please add a note to the item if the item is not applicable and please add links to any relevant follow up issues.* I have... - [x] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [x] added `!` to the type prefix if API or client breaking change - [x] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting)) - [x] provided a link to the relevant issue or specification - [x] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules) - [x] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing) - [x] added a changelog entry to `CHANGELOG.md` - N/A imo - [x] included comments for [documenting Go code](https://blog.golang.org/godoc) - [x] updated the relevant documentation or specification - [x] reviewed "Files changed" and left comments if necessary - [x] confirmed all CI checks have passed - lint failure unrelated ### Reviewers Checklist *All items are required. Please add a note if the item is not applicable and please add your handle next to the items reviewed if you only reviewed selected items.* I have... - [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] confirmed `!` in the type prefix if API or client breaking change - [ ] confirmed all author checklist items have been addressed - [ ] reviewed state machine logic - [ ] reviewed API design and naming - [ ] reviewed documentation is accurate - [ ] reviewed tests and test coverage - [ ] manually tested (if applicable) --- store/cachekv/bench_helper_test.go | 44 ++++++++++++++ store/cachekv/store_bench_test.go | 92 ++++++++++++++++++++++-------- 2 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 store/cachekv/bench_helper_test.go diff --git a/store/cachekv/bench_helper_test.go b/store/cachekv/bench_helper_test.go new file mode 100644 index 000000000000..fe5be27fabc9 --- /dev/null +++ b/store/cachekv/bench_helper_test.go @@ -0,0 +1,44 @@ +package cachekv_test + +import "crypto/rand" + +func randSlice(sliceSize int) []byte { + bz := make([]byte, sliceSize) + _, _ = rand.Read(bz) + return bz +} + +func incrementByteSlice(bz []byte) { + for index := len(bz) - 1; index >= 0; index-- { + if bz[index] < 255 { + bz[index]++ + break + } else { + bz[index] = 0 + } + } +} + +// Generate many keys starting at startKey, and are in sequential order +func generateSequentialKeys(startKey []byte, numKeys int) [][]byte { + toReturn := make([][]byte, 0, numKeys) + cur := make([]byte, len(startKey)) + copy(cur, startKey) + for i := 0; i < numKeys; i++ { + newKey := make([]byte, len(startKey)) + copy(newKey, cur) + toReturn = append(toReturn, newKey) + incrementByteSlice(cur) + } + return toReturn +} + +// Generate many random, unsorted keys +func generateRandomKeys(keySize int, numKeys int) [][]byte { + toReturn := make([][]byte, 0, numKeys) + for i := 0; i < numKeys; i++ { + newKey := randSlice(keySize) + toReturn = append(toReturn, newKey) + } + return toReturn +} diff --git a/store/cachekv/store_bench_test.go b/store/cachekv/store_bench_test.go index 2957fe6a6503..040c4a77c97c 100644 --- a/store/cachekv/store_bench_test.go +++ b/store/cachekv/store_bench_test.go @@ -1,8 +1,6 @@ package cachekv_test import ( - "crypto/rand" - "sort" "testing" dbm "github.com/tendermint/tm-db" @@ -11,37 +9,85 @@ import ( "github.com/cosmos/cosmos-sdk/store/dbadapter" ) -func benchmarkCacheKVStoreIterator(numKVs int, b *testing.B) { +var sink interface{} + +const defaultValueSizeBz = 1 << 12 + +// This benchmark measures the time of iterator.Next() when the parent store is blank +func benchmarkBlankParentIteratorNext(b *testing.B, keysize int) { + mem := dbadapter.Store{DB: dbm.NewMemDB()} + kvstore := cachekv.NewStore(mem) + // Use a singleton for value, to not waste time computing it + value := randSlice(defaultValueSizeBz) + // Use simple values for keys, pick a random start, + // and take next b.N keys sequentially after.] + startKey := randSlice(32) + + // Add 1 to avoid issues when b.N = 1 + keys := generateSequentialKeys(startKey, b.N+1) + for _, k := range keys { + kvstore.Set(k, value) + } + b.ReportAllocs() + b.ResetTimer() + + iter := kvstore.Iterator(keys[0], keys[b.N]) + defer iter.Close() + + for _ = iter.Key(); iter.Valid(); iter.Next() { + // deadcode elimination stub + sink = iter + } +} + +// Benchmark setting New keys to a store, where the new keys are in sequence. +func benchmarkBlankParentAppend(b *testing.B, keysize int) { mem := dbadapter.Store{DB: dbm.NewMemDB()} - cstore := cachekv.NewStore(mem) - keys := make([]string, numKVs) + kvstore := cachekv.NewStore(mem) - for i := 0; i < numKVs; i++ { - key := make([]byte, 32) - value := make([]byte, 32) + // Use a singleton for value, to not waste time computing it + value := randSlice(32) + // Use simple values for keys, pick a random start, + // and take next b.N keys sequentially after. + startKey := randSlice(32) - _, _ = rand.Read(key) - _, _ = rand.Read(value) + keys := generateSequentialKeys(startKey, b.N) - keys[i] = string(key) - cstore.Set(key, value) + b.ReportAllocs() + b.ResetTimer() + + for _, k := range keys { + kvstore.Set(k, value) } +} - sort.Strings(keys) +// Benchmark setting New keys to a store, where the new keys are random. +// the speed of this function does not depend on the values in the parent store +func benchmarkRandomSet(b *testing.B, keysize int) { + mem := dbadapter.Store{DB: dbm.NewMemDB()} + kvstore := cachekv.NewStore(mem) - for n := 0; n < b.N; n++ { - iter := cstore.Iterator([]byte(keys[0]), []byte(keys[numKVs-1])) + // Use a singleton for value, to not waste time computing it + value := randSlice(defaultValueSizeBz) + keys := generateRandomKeys(keysize, b.N) - for _ = iter.Key(); iter.Valid(); iter.Next() { - } + b.ReportAllocs() + b.ResetTimer() - iter.Close() + for _, k := range keys { + kvstore.Set(k, value) } } -func BenchmarkCacheKVStoreIterator500(b *testing.B) { benchmarkCacheKVStoreIterator(500, b) } -func BenchmarkCacheKVStoreIterator1000(b *testing.B) { benchmarkCacheKVStoreIterator(1000, b) } -func BenchmarkCacheKVStoreIterator10000(b *testing.B) { benchmarkCacheKVStoreIterator(10000, b) } -func BenchmarkCacheKVStoreIterator50000(b *testing.B) { benchmarkCacheKVStoreIterator(50000, b) } -func BenchmarkCacheKVStoreIterator100000(b *testing.B) { benchmarkCacheKVStoreIterator(100000, b) } +func BenchmarkBlankParentIteratorNextKeySize32(b *testing.B) { + benchmarkBlankParentIteratorNext(b, 32) +} + +func BenchmarkBlankParentAppendKeySize32(b *testing.B) { + benchmarkBlankParentAppend(b, 32) +} + +func BenchmarkSetKeySize32(b *testing.B) { + benchmarkRandomSet(b, 32) +}