diff --git a/FORKED_CHANGELOG.md b/FORKED_CHANGELOG.md index e1a5ebbcb..1f149dd71 100644 --- a/FORKED_CHANGELOG.md +++ b/FORKED_CHANGELOG.md @@ -48,6 +48,8 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [\#1](https://github.com/sei-protocol/sei-cosmos/pull/1) Integrate Cosmos with sei-tendermint and ABCI++ * [\#14](https://github.com/sei-protocol/sei-cosmos/pull/14) Integrate Cosmos with Tendermint tracing * (x/auth/vesting) [\#11652](https://github.com/cosmos/cosmos-sdk/pull/11652) Add util functions for `Period(s)` +* [\#14168](https://github.com/cosmos/cosmos-sdk/pull/14168) perf: store/cachekv: preallocate kvL in dirtyItems which gets appended too +* [\#10024](https://github.com/cosmos/cosmos-sdk/pull/10024) fix!: store/cachekv: reduce growth factor for iterator ranging using binary searches #10024 ### Features * [\#17](https://github.com/sei-protocol/sei-cosmos/pull/17) Support SR25519 algorithm for client transaction signing diff --git a/go.mod b/go.mod index 27a592dcb..201037ff5 100644 --- a/go.mod +++ b/go.mod @@ -140,7 +140,7 @@ replace ( github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.7.0 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 - github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.1.94 + github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.1.102 // latest grpc doesn't work with with our modified proto compiler, so we need to enforce // the following version across all dependencies. diff --git a/go.sum b/go.sum index 86d630b6d..18bc4f071 100644 --- a/go.sum +++ b/go.sum @@ -684,8 +684,8 @@ github.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8 h1:ajJQhvqPSQFJJ4aV5mDAM github.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8/go.mod h1:Nw/CCOXNyF5JDd6UpYxBwG5WWZ2FOJ/d5QnXL4KQ6vY= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= -github.com/sei-protocol/sei-tendermint v0.1.94 h1:YlAU7w2s7tDqPez8Xmg2d3ImrjTPeCyo8eTLtabjODY= -github.com/sei-protocol/sei-tendermint v0.1.94/go.mod h1:Olwbjyagrpoxj5DAUhHxMTWDVEfQ3FYdpypaJ3+6Hs8= +github.com/sei-protocol/sei-tendermint v0.1.102 h1:YwUfpwuZiUvyxhQTvTp/NV0fZWDG+QvemLB+3wTjE1k= +github.com/sei-protocol/sei-tendermint v0.1.102/go.mod h1:Olwbjyagrpoxj5DAUhHxMTWDVEfQ3FYdpypaJ3+6Hs8= github.com/sei-protocol/sei-tm-db v0.0.5 h1:3WONKdSXEqdZZeLuWYfK5hP37TJpfaUa13vAyAlvaQY= github.com/sei-protocol/sei-tm-db v0.0.5/go.mod h1:Cpa6rGyczgthq7/0pI31jys2Fw0Nfrc+/jKdP1prVqY= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= diff --git a/store/cachekv/memiterator.go b/store/cachekv/memiterator.go index c46b72cc9..96be270b0 100644 --- a/store/cachekv/memiterator.go +++ b/store/cachekv/memiterator.go @@ -45,7 +45,6 @@ func newMemIterator( return &memIterator{ Iterator: iter, - lastKey: nil, deleted: deleted, eventManager: eventManager, @@ -64,7 +63,6 @@ func (mi *memIterator) Value() []byte { if _, ok := mi.deleted.Load(string(key)); ok && !reCallingOnOldLastKey { return nil } - value := mi.Iterator.Value() mi.eventManager.EmitResourceAccessReadEvent("iterator", mi.storeKey, key, value) diff --git a/store/cachekv/store.go b/store/cachekv/store.go index fdf4df24c..16e71d411 100644 --- a/store/cachekv/store.go +++ b/store/cachekv/store.go @@ -237,10 +237,8 @@ func (store *Store) iterator(start, end []byte, ascending bool) types.Iterator { } else { parent = store.parent.ReverseIterator(start, end) } - store.dirtyItems(start, end) cache = newMemIterator(start, end, store.sortedCache, store.deleted, ascending, store.eventManager, store.storeKey) - return NewCacheMergeIterator(parent, cache, ascending) } @@ -329,7 +327,7 @@ const minSortSize = 1024 // Constructs a slice of dirty items, to use w/ memIterator. func (store *Store) dirtyItems(start, end []byte) { startStr, endStr := conv.UnsafeBytesToStr(start), conv.UnsafeBytesToStr(end) - if startStr > endStr { + if end != nil && startStr > endStr { // Nothing to do here. return } @@ -342,6 +340,7 @@ func (store *Store) dirtyItems(start, end []byte) { // O(N^2) overhead. // Even without that, too many range checks eventually becomes more expensive // than just not having the cache. + store.emitUnsortedCacheSizeMetric() if n < minSortSize { for key := range store.unsortedCache { if dbm.IsKeyInDomain(conv.UnsafeStrToBytes(key), start, end) { @@ -374,7 +373,7 @@ func (store *Store) dirtyItems(start, end []byte) { } } - kvL := make([]*kv.Pair, 0) + kvL := make([]*kv.Pair, 0, 1+endIndex-startIndex) for i := startIndex; i <= endIndex; i++ { key := strL[i] cacheValue, _ := store.cache.Get(key) @@ -383,6 +382,12 @@ func (store *Store) dirtyItems(start, end []byte) { // kvL was already sorted so pass it in as is. store.clearUnsortedCacheSubset(kvL, stateAlreadySorted) + store.emitUnsortedCacheSizeMetric() +} + +func (store *Store) emitUnsortedCacheSizeMetric() { + n := len(store.unsortedCache) + telemetry.SetGauge(float32(n), "sei", "cosmos", "unsorted", "cache", "size") } func findStartEndIndex(strL []string, startStr, endStr string) (int, int) { @@ -413,12 +418,13 @@ func (store *Store) clearUnsortedCacheSubset(unsorted []*kv.Pair, sortState sort if item.Value == nil { // deleted element, tracked by store.deleted // setting arbitrary value - // TODO: Don't ignore this error. - store.sortedCache.Set(item.Key, []byte{}) + if err := store.sortedCache.Set(item.Key, []byte{}); err != nil { + panic(err) + } + continue } - err := store.sortedCache.Set(item.Key, item.Value) - if err != nil { + if err := store.sortedCache.Set(item.Key, item.Value); err != nil { panic(err) } } @@ -426,6 +432,7 @@ func (store *Store) clearUnsortedCacheSubset(unsorted []*kv.Pair, sortState sort func (store *Store) deleteKeysFromUnsortedCache(unsorted []*kv.Pair) { n := len(store.unsortedCache) + store.emitUnsortedCacheSizeMetric() if len(unsorted) == n { // This pattern allows the Go compiler to emit the map clearing idiom for the entire map. for key := range store.unsortedCache { delete(store.unsortedCache, key) @@ -435,6 +442,7 @@ func (store *Store) deleteKeysFromUnsortedCache(unsorted []*kv.Pair) { delete(store.unsortedCache, conv.UnsafeBytesToStr(kv.Key)) } } + defer store.emitUnsortedCacheSizeMetric() } //----------------------------------------