chore: Unfork iavl (cosmos#377)

* bump iavl

* add missing skipFastStorageUpgrade to function call

* fix tree.Set calls

* update Tree interface and immutableTree implementation

* fix TestImmutableTreePanics

* update iavl store implementation

* fix TestImmutableTreePanics

* add LoadVersionForOverwriting and GetAllVersions to iavl store

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/confio/ics23/go v0.7.0
 	github.com/cosmos/btcutil v1.0.4
 	github.com/cosmos/go-bip39 v1.0.0
-	github.com/cosmos/iavl v0.17.3
+	github.com/cosmos/iavl v0.19.4
 	github.com/cosmos/ledger-cosmos-go v0.11.1
 	github.com/gogo/gateway v1.1.0
 	github.com/gogo/protobuf v1.3.3
@@ -125,7 +125,6 @@ require (
 
 replace (
 	github.com/99designs/keyring => github.com/cosmos/keyring v1.1.7-0.20210622111912-ef00f8ac3d76
-	github.com/cosmos/iavl => github.com/osmosis-labs/iavl v0.17.3-osmo-v7
 
 	// Fix upstream GHSA-h395-qcrw-5vmq vulnerability.
 	// TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409

store/cache/cache_test.go

@@ -18,7 +18,7 @@ func TestGetOrSetStoreCache(t *testing.T) {
 	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
 
 	sKey := types.NewKVStoreKey("test")
-	tree, err := iavl.NewMutableTree(db, 100)
+	tree, err := iavl.NewMutableTree(db, 100, false)
 	require.NoError(t, err)
 	store := iavlstore.UnsafeNewStore(tree)
 	store2 := mngr.GetStoreCache(sKey, store)
@@ -32,7 +32,7 @@ func TestUnwrap(t *testing.T) {
 	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
 
 	sKey := types.NewKVStoreKey("test")
-	tree, err := iavl.NewMutableTree(db, 100)
+	tree, err := iavl.NewMutableTree(db, 100, false)
 	require.NoError(t, err)
 	store := iavlstore.UnsafeNewStore(tree)
 	_ = mngr.GetStoreCache(sKey, store)
@@ -46,7 +46,7 @@ func TestStoreCache(t *testing.T) {
 	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
 
 	sKey := types.NewKVStoreKey("test")
-	tree, err := iavl.NewMutableTree(db, 100)
+	tree, err := iavl.NewMutableTree(db, 100, false)
 	require.NoError(t, err)
 	store := iavlstore.UnsafeNewStore(tree)
 	kvStore := mngr.GetStoreCache(sKey, store)

store/iavl/store.go

@@ -53,12 +53,17 @@ func LoadStore(db dbm.DB, logger log.Logger, key types.StoreKey, id types.Commit
 // provided DB. An error is returned if the version fails to load, or if called with a positive
 // version on an empty tree.
 func LoadStoreWithInitialVersion(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, initialVersion uint64, cacheSize int) (types.CommitKVStore, error) {
-	tree, err := iavl.NewMutableTreeWithOpts(db, cacheSize, &iavl.Options{InitialVersion: initialVersion})
+	tree, err := iavl.NewMutableTreeWithOpts(db, cacheSize, &iavl.Options{InitialVersion: initialVersion}, false)
 	if err != nil {
 		return nil, err
 	}
 
-	if tree.IsUpgradeable() && logger != nil {
+	upgradable, err := tree.IsUpgradeable()
+	if err != nil {
+		return nil, err
+	}
+
+	if upgradable && logger != nil {
 		logger.Info(
 			"Upgrading IAVL storage for faster queries + execution on live state. This may take a while",
 			"store_key", key.String(),
@@ -137,9 +142,14 @@ func (st *Store) Commit() types.CommitID {
 
 // LastCommitID implements Committer.
 func (st *Store) LastCommitID() types.CommitID {
+	hash, err := st.tree.Hash()
+	if err != nil {
+		panic(err)
+	}
+
 	return types.CommitID{
 		Version: st.tree.Version(),
-		Hash:    st.tree.Hash(),
+		Hash:    hash,
 	}
 }
 
@@ -160,6 +170,11 @@ func (st *Store) VersionExists(version int64) bool {
 	return st.tree.VersionExists(version)
 }
 
+// GetAllVersions returns all versions in the iavl tree
+func (st *Store) GetAllVersions() []int {
+	return st.tree.AvailableVersions()
+}
+
 // Implements Store.
 func (st *Store) GetStoreType() types.StoreType {
 	return types.StoreTypeIAVL
@@ -190,13 +205,21 @@ func (st *Store) Set(key, value []byte) {
 // Implements types.KVStore.
 func (st *Store) Get(key []byte) []byte {
 	defer telemetry.MeasureSince(time.Now(), "store", "iavl", "get")
-	return st.tree.Get(key)
+	value, err := st.tree.Get(key)
+	if err != nil {
+		panic(err)
+	}
+	return value
 }
 
 // Implements types.KVStore.
 func (st *Store) Has(key []byte) (exists bool) {
 	defer telemetry.MeasureSince(time.Now(), "store", "iavl", "has")
-	return st.tree.Has(key)
+	has, err := st.tree.Has(key)
+	if err != nil {
+		panic(err)
+	}
+	return has
 }
 
 // Implements types.KVStore.
@@ -212,20 +235,34 @@ func (st *Store) DeleteVersions(versions ...int64) error {
 	return st.tree.DeleteVersions(versions...)
 }
 
+// LoadVersionForOverwriting attempts to load a tree at a previously committed
+// version, or the latest version below it. Any versions greater than targetVersion will be deleted.
+func (st *Store) LoadVersionForOverwriting(targetVersion int64) (int64, error) {
+	return st.tree.LoadVersionForOverwriting(targetVersion)
+}
+
 // Implements types.KVStore.
 // CONTRACT: Caller must release the iavlIterator, as each one creates a new
 // goroutine.
 // CONTRACT: There must be no writes to the store while an iterator is not closed.
 func (st *Store) Iterator(start, end []byte) types.Iterator {
-	return st.tree.Iterator(start, end, true)
+	iterator, err := st.tree.Iterator(start, end, true)
+	if err != nil {
+		panic(err)
+	}
+	return iterator
 }
 
 // Implements types.KVStore.
 // CONTRACT: Caller must release the iavlIterator, as each one creates a new
 // goroutine.
 // CONTRACT: There must be no writes to the store while an iterator is not closed.
 func (st *Store) ReverseIterator(start, end []byte) types.Iterator {
-	return st.tree.Iterator(start, end, false)
+	iterator, err := st.tree.Iterator(start, end, false)
+	if err != nil {
+		panic(err)
+	}
+	return iterator
 }
 
 // SetInitialVersion sets the initial version of the IAVL tree. It is used when
@@ -300,7 +337,12 @@ func (st *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) {
 			break
 		}
 
-		res.Value = tree.GetVersioned(key, res.Height)
+		value, err := tree.GetVersioned(key, res.Height)
+		if err != nil {
+			panic(err)
+		}
+		res.Value = value
+
 		if !req.Prove {
 			break
 		}

store/iavl/store_test.go

@@ -3,9 +3,10 @@ package iavl
 import (
 	crand "crypto/rand"
 	"fmt"
-	"github.com/tendermint/tendermint/libs/log"
 	"testing"
 
+	"github.com/tendermint/tendermint/libs/log"
+
 	"github.com/cosmos/cosmos-sdk/store/cachekv"
 
 	"github.com/cosmos/iavl"
@@ -34,7 +35,7 @@ func randBytes(numBytes int) []byte {
 
 // make a tree with data from above and save it
 func newAlohaTree(t *testing.T, db dbm.DB) (*iavl.MutableTree, types.CommitID) {
-	tree, err := iavl.NewMutableTree(db, cacheSize)
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
 	require.NoError(t, err)
 
 	for k, v := range treeData {
@@ -59,21 +60,30 @@ func TestLoadStore(t *testing.T) {
 	store := UnsafeNewStore(tree)
 
 	// Create non-pruned height H
-	require.True(t, tree.Set([]byte("hello"), []byte("hallo")))
+	updated, err := tree.Set([]byte("hello"), []byte("hallo"))
+
+	require.True(t, updated)
+	require.NoError(t, err)
 	hash, verH, err := tree.SaveVersion()
 	cIDH := types.CommitID{Version: verH, Hash: hash}
 	require.Nil(t, err)
 
 	// Create pruned height Hp
-	require.True(t, tree.Set([]byte("hello"), []byte("hola")))
+	updated, err = tree.Set([]byte("hello"), []byte("hola"))
+
+	require.True(t, updated)
+	require.NoError(t, err)
 	hash, verHp, err := tree.SaveVersion()
 	cIDHp := types.CommitID{Version: verHp, Hash: hash}
 	require.Nil(t, err)
 
 	// TODO: Prune this height
 
 	// Create current height Hc
-	require.True(t, tree.Set([]byte("hello"), []byte("ciao")))
+	updated, err = tree.Set([]byte("hello"), []byte("ciao"))
+
+	require.True(t, updated)
+	require.NoError(t, err)
 	hash, verHc, err := tree.SaveVersion()
 	cIDHc := types.CommitID{Version: verHc, Hash: hash}
 	require.Nil(t, err)
@@ -114,7 +124,9 @@ func TestGetImmutable(t *testing.T) {
 	tree, cID := newAlohaTree(t, db)
 	store := UnsafeNewStore(tree)
 
-	require.True(t, tree.Set([]byte("hello"), []byte("adios")))
+	updated, err := tree.Set([]byte("hello"), []byte("adios"))
+	require.True(t, updated)
+	require.NoError(t, err)
 	hash, ver, err := tree.SaveVersion()
 	cID = types.CommitID{Version: ver, Hash: hash}
 	require.Nil(t, err)
@@ -274,7 +286,7 @@ func TestIAVLIterator(t *testing.T) {
 func TestIAVLReverseIterator(t *testing.T) {
 	db := dbm.NewMemDB()
 
-	tree, err := iavl.NewMutableTree(db, cacheSize)
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
 	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
@@ -307,7 +319,7 @@ func TestIAVLReverseIterator(t *testing.T) {
 
 func TestIAVLPrefixIterator(t *testing.T) {
 	db := dbm.NewMemDB()
-	tree, err := iavl.NewMutableTree(db, cacheSize)
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
 	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
@@ -371,7 +383,7 @@ func TestIAVLPrefixIterator(t *testing.T) {
 
 func TestIAVLReversePrefixIterator(t *testing.T) {
 	db := dbm.NewMemDB()
-	tree, err := iavl.NewMutableTree(db, cacheSize)
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
 	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
@@ -439,7 +451,7 @@ func nextVersion(iavl *Store) {
 
 func TestIAVLNoPrune(t *testing.T) {
 	db := dbm.NewMemDB()
-	tree, err := iavl.NewMutableTree(db, cacheSize)
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
 	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
@@ -458,7 +470,7 @@ func TestIAVLNoPrune(t *testing.T) {
 
 func TestIAVLStoreQuery(t *testing.T) {
 	db := dbm.NewMemDB()
-	tree, err := iavl.NewMutableTree(db, cacheSize)
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
 	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
@@ -562,7 +574,7 @@ func BenchmarkIAVLIteratorNext(b *testing.B) {
 	b.ReportAllocs()
 	db := dbm.NewMemDB()
 	treeSize := 1000
-	tree, err := iavl.NewMutableTree(db, cacheSize)
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
 	require.NoError(b, err)
 
 	for i := 0; i < treeSize; i++ {
@@ -596,7 +608,7 @@ func TestSetInitialVersion(t *testing.T) {
 		{
 			"works with a mutable tree",
 			func(db *dbm.MemDB) *Store {
-				tree, err := iavl.NewMutableTree(db, cacheSize)
+				tree, err := iavl.NewMutableTree(db, cacheSize, false)
 				require.NoError(t, err)
 				store := UnsafeNewStore(tree)
 
@@ -606,7 +618,7 @@ func TestSetInitialVersion(t *testing.T) {
 		{
 			"throws error on immutable tree",
 			func(db *dbm.MemDB) *Store {
-				tree, err := iavl.NewMutableTree(db, cacheSize)
+				tree, err := iavl.NewMutableTree(db, cacheSize, false)
 				require.NoError(t, err)
 				store := UnsafeNewStore(tree)
 				_, version, err := store.tree.SaveVersion()

store/iavl/tree.go

@@ -18,21 +18,23 @@ type (
 	// implemented by an iavl.MutableTree. For an immutable IAVL tree, a wrapper
 	// must be made.
 	Tree interface {
-		Has(key []byte) bool
-		Get(key []byte) []byte
-		Set(key, value []byte) bool
-		Remove(key []byte) ([]byte, bool)
+		Has(key []byte) (bool, error)
+		Get(key []byte) ([]byte, error)
+		Set(key, value []byte) (bool, error)
+		Remove(key []byte) ([]byte, bool, error)
 		SaveVersion() ([]byte, int64, error)
 		DeleteVersion(version int64) error
 		DeleteVersions(versions ...int64) error
 		Version() int64
-		Hash() []byte
+		Hash() ([]byte, error)
 		VersionExists(version int64) bool
-		GetVersioned(key []byte, version int64) []byte
+		GetVersioned(key []byte, version int64) ([]byte, error)
 		GetVersionedWithProof(key []byte, version int64) ([]byte, *iavl.RangeProof, error)
 		GetImmutable(version int64) (*iavl.ImmutableTree, error)
 		SetInitialVersion(version uint64)
-		Iterator(start, end []byte, ascending bool) types.Iterator
+		Iterator(start, end []byte, ascending bool) (types.Iterator, error)
+		AvailableVersions() []int
+		LoadVersionForOverwriting(targetVersion int64) (int64, error)
 	}
 
 	// immutableTree is a simple wrapper around a reference to an iavl.ImmutableTree
@@ -43,11 +45,11 @@ type (
 	}
 )
 
-func (it *immutableTree) Set(_, _ []byte) bool {
+func (it *immutableTree) Set(_, _ []byte) (bool, error) {
 	panic("cannot call 'Set' on an immutable IAVL tree")
 }
 
-func (it *immutableTree) Remove(_ []byte) ([]byte, bool) {
+func (it *immutableTree) Remove(_ []byte) ([]byte, bool, error) {
 	panic("cannot call 'Remove' on an immutable IAVL tree")
 }
 
@@ -71,9 +73,9 @@ func (it *immutableTree) VersionExists(version int64) bool {
 	return it.Version() == version
 }
 
-func (it *immutableTree) GetVersioned(key []byte, version int64) []byte {
+func (it *immutableTree) GetVersioned(key []byte, version int64) ([]byte, error) {
 	if it.Version() != version {
-		return nil
+		return nil, fmt.Errorf("version mismatch on immutable IAVL tree; got: %d, expected: %d", version, it.Version())
 	}
 
 	return it.Get(key)
@@ -94,3 +96,11 @@ func (it *immutableTree) GetImmutable(version int64) (*iavl.ImmutableTree, error
 
 	return it.ImmutableTree, nil
 }
+
+func (it *immutableTree) AvailableVersions() []int {
+	return []int{}
+}
+
+func (it *immutableTree) LoadVersionForOverwriting(targetVersion int64) (int64, error) {
+	panic("cannot call 'LoadVersionForOverwriting' on an immutable IAVL tree")
+}

store/iavl/tree_test.go

@@ -10,15 +10,17 @@ import (
 
 func TestImmutableTreePanics(t *testing.T) {
 	t.Parallel()
-	immTree := iavl.NewImmutableTree(dbm.NewMemDB(), 100)
+	immTree := iavl.NewImmutableTree(dbm.NewMemDB(), 100, false)
 	it := &immutableTree{immTree}
 	require.Panics(t, func() { it.Set([]byte{}, []byte{}) })
 	require.Panics(t, func() { it.Remove([]byte{}) })
 	require.Panics(t, func() { it.SaveVersion() })           // nolint:errcheck
 	require.Panics(t, func() { it.DeleteVersion(int64(1)) }) // nolint:errcheck
-	v := it.GetVersioned([]byte{0x01}, 1)
+	v, err := it.GetVersioned([]byte{0x01}, 1)
+	require.Error(t, err)
 	require.Nil(t, v)
-	v = it.GetVersioned([]byte{0x01}, 0)
+	v, err = it.GetVersioned([]byte{0x01}, 0)
+	require.NoError(t, err)
 	require.Nil(t, v)
 
 	val, proof, err := it.GetVersionedWithProof(nil, 1)