CacheKv optimizations (sei-protocol#113)

## Describe your changes and provide context Cherry picking changes here: * [\#14168](cosmos/cosmos-sdk#14168) perf: store/cachekv: preallocate kvL in dirtyItems which gets appended too * [\#10024](cosmos/cosmos-sdk#10024) fix!: store/cachekv: reduce growth factor for iterator ranging using binary searches #10024 ## Testing performed to validate your change Ran a cluster ![image](https://user-images.githubusercontent.com/18161326/207636380-1c404827-cdb4-4bdc-aade-bfc0294f6cd4.png)
Timwood0x10 · Dec 15, 2022 · a7c6045 · a7c6045
1 parent dc36a89
commit a7c6045
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 13 deletions.
diff --git a/FORKED_CHANGELOG.md b/FORKED_CHANGELOG.md
@@ -48,6 +48,8 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * [\#1](https://github.com/sei-protocol/sei-cosmos/pull/1) Integrate Cosmos with sei-tendermint and ABCI++
 * [\#14](https://github.com/sei-protocol/sei-cosmos/pull/14) Integrate Cosmos with Tendermint tracing
 * (x/auth/vesting) [\#11652](https://github.com/cosmos/cosmos-sdk/pull/11652) Add util functions for `Period(s)`
+* [\#14168](https://github.com/cosmos/cosmos-sdk/pull/14168) perf: store/cachekv: preallocate kvL in dirtyItems which gets appended too
+* [\#10024](https://github.com/cosmos/cosmos-sdk/pull/10024) fix!: store/cachekv: reduce growth factor for iterator ranging using binary searches #10024
 
 ### Features
 * [\#17](https://github.com/sei-protocol/sei-cosmos/pull/17) Support SR25519 algorithm for client transaction signing

diff --git a/go.mod b/go.mod
@@ -140,7 +140,7 @@ replace (
 	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.7.0
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 
-	github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.1.94
+	github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.1.102
 
 	// latest grpc doesn't work with with our modified proto compiler, so we need to enforce
 	// the following version across all dependencies.

diff --git a/go.sum b/go.sum
@@ -684,8 +684,8 @@ github.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8 h1:ajJQhvqPSQFJJ4aV5mDAM
 github.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8/go.mod h1:Nw/CCOXNyF5JDd6UpYxBwG5WWZ2FOJ/d5QnXL4KQ6vY=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
-github.com/sei-protocol/sei-tendermint v0.1.94 h1:YlAU7w2s7tDqPez8Xmg2d3ImrjTPeCyo8eTLtabjODY=
-github.com/sei-protocol/sei-tendermint v0.1.94/go.mod h1:Olwbjyagrpoxj5DAUhHxMTWDVEfQ3FYdpypaJ3+6Hs8=
+github.com/sei-protocol/sei-tendermint v0.1.102 h1:YwUfpwuZiUvyxhQTvTp/NV0fZWDG+QvemLB+3wTjE1k=
+github.com/sei-protocol/sei-tendermint v0.1.102/go.mod h1:Olwbjyagrpoxj5DAUhHxMTWDVEfQ3FYdpypaJ3+6Hs8=
 github.com/sei-protocol/sei-tm-db v0.0.5 h1:3WONKdSXEqdZZeLuWYfK5hP37TJpfaUa13vAyAlvaQY=
 github.com/sei-protocol/sei-tm-db v0.0.5/go.mod h1:Cpa6rGyczgthq7/0pI31jys2Fw0Nfrc+/jKdP1prVqY=
 github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=

diff --git a/store/cachekv/memiterator.go b/store/cachekv/memiterator.go
@@ -45,7 +45,6 @@ func newMemIterator(
 
 	return &memIterator{
 		Iterator: iter,
-
 		lastKey: nil,
 		deleted: deleted,
 		eventManager: eventManager,
@@ -64,7 +63,6 @@ func (mi *memIterator) Value() []byte {
 	if _, ok := mi.deleted.Load(string(key)); ok && !reCallingOnOldLastKey {
 		return nil
 	}
-
 	value := mi.Iterator.Value()
 	mi.eventManager.EmitResourceAccessReadEvent("iterator", mi.storeKey, key, value)
 

diff --git a/store/cachekv/store.go b/store/cachekv/store.go
@@ -237,10 +237,8 @@ func (store *Store) iterator(start, end []byte, ascending bool) types.Iterator {
 	} else {
 		parent = store.parent.ReverseIterator(start, end)
 	}
-
 	store.dirtyItems(start, end)
 	cache = newMemIterator(start, end, store.sortedCache, store.deleted, ascending, store.eventManager, store.storeKey)
-
 	return NewCacheMergeIterator(parent, cache, ascending)
 }
 
@@ -329,7 +327,7 @@ const minSortSize = 1024
 // Constructs a slice of dirty items, to use w/ memIterator.
 func (store *Store) dirtyItems(start, end []byte) {
 	startStr, endStr := conv.UnsafeBytesToStr(start), conv.UnsafeBytesToStr(end)
-	if startStr > endStr {
+	if end != nil && startStr > endStr {
 		// Nothing to do here.
 		return
 	}
@@ -342,6 +340,7 @@ func (store *Store) dirtyItems(start, end []byte) {
 	// O(N^2) overhead.
 	// Even without that, too many range checks eventually becomes more expensive
 	// than just not having the cache.
+	store.emitUnsortedCacheSizeMetric()
 	if n < minSortSize {
 		for key := range store.unsortedCache {
 			if dbm.IsKeyInDomain(conv.UnsafeStrToBytes(key), start, end) {
@@ -374,7 +373,7 @@ func (store *Store) dirtyItems(start, end []byte) {
 		}
 	}
 
-	kvL := make([]*kv.Pair, 0)
+	kvL := make([]*kv.Pair, 0, 1+endIndex-startIndex)
 	for i := startIndex; i <= endIndex; i++ {
 		key := strL[i]
 		cacheValue, _ := store.cache.Get(key)
@@ -383,6 +382,12 @@ func (store *Store) dirtyItems(start, end []byte) {
 
 	// kvL was already sorted so pass it in as is.
 	store.clearUnsortedCacheSubset(kvL, stateAlreadySorted)
+	store.emitUnsortedCacheSizeMetric()
+}
+
+func (store *Store) emitUnsortedCacheSizeMetric() {
+	n := len(store.unsortedCache)
+	telemetry.SetGauge(float32(n), "sei", "cosmos", "unsorted", "cache", "size")
 }
 
 func findStartEndIndex(strL []string, startStr, endStr string) (int, int) {
@@ -413,19 +418,21 @@ func (store *Store) clearUnsortedCacheSubset(unsorted []*kv.Pair, sortState sort
 		if item.Value == nil {
 			// deleted element, tracked by store.deleted
 			// setting arbitrary value
-			// TODO: Don't ignore this error.
-			store.sortedCache.Set(item.Key, []byte{})
+			if err := store.sortedCache.Set(item.Key, []byte{}); err != nil {
+				panic(err)
+			}
+
 			continue
 		}
-		err := store.sortedCache.Set(item.Key, item.Value)
-		if err != nil {
+		if err := store.sortedCache.Set(item.Key, item.Value); err != nil {
 			panic(err)
 		}
 	}
 }
 
 func (store *Store) deleteKeysFromUnsortedCache(unsorted []*kv.Pair) {
 	n := len(store.unsortedCache)
+	store.emitUnsortedCacheSizeMetric()
 	if len(unsorted) == n { // This pattern allows the Go compiler to emit the map clearing idiom for the entire map.
 		for key := range store.unsortedCache {
 			delete(store.unsortedCache, key)
@@ -435,6 +442,7 @@ func (store *Store) deleteKeysFromUnsortedCache(unsorted []*kv.Pair) {
 			delete(store.unsortedCache, conv.UnsafeBytesToStr(kv.Key))
 		}
 	}
+	defer store.emitUnsortedCacheSizeMetric()
 }
 
 //----------------------------------------