Refactor IAVL Pruning

baseapp/baseapp_test.go

@@ -151,7 +151,7 @@ func useFileUpgradeLoader(upgradeInfoPath string) func(*BaseApp) {
 
 func initStore(t *testing.T, db dbm.DB, storeKey string, k, v []byte) {
 	rs := rootmulti.NewStore(db)
-	rs.SetPruning(store.PruneSyncable)
+	rs.SetPruning(store.PruneNothing)
 	key := sdk.NewKVStoreKey(storeKey)
 	rs.MountStoreWithDB(key, store.StoreTypeIAVL, nil)
 	err := rs.LoadLatestVersion()
@@ -244,7 +244,7 @@ func TestSetLoader(t *testing.T) {
 			initStore(t, db, tc.origStoreKey, k, v)
 
 			// load the app with the existing db
-			opts := []func(*BaseApp){SetPruning(store.PruneSyncable)}
+			opts := []func(*BaseApp){SetPruning(store.PruneNothing)}
 			if tc.setLoader != nil {
 				opts = append(opts, tc.setLoader)
 			}

store/cache/cache_test.go

@@ -20,7 +20,7 @@ func TestGetOrSetStoreCache(t *testing.T) {
 	sKey := types.NewKVStoreKey("test")
 	tree, err := iavl.NewMutableTree(db, 100)
 	require.NoError(t, err)
-	store := iavlstore.UnsafeNewStore(tree, 10, 10)
+	store := iavlstore.UnsafeNewStore(tree)
 	store2 := mngr.GetStoreCache(sKey, store)
 
 	require.NotNil(t, store2)
@@ -34,7 +34,7 @@ func TestUnwrap(t *testing.T) {
 	sKey := types.NewKVStoreKey("test")
 	tree, err := iavl.NewMutableTree(db, 100)
 	require.NoError(t, err)
-	store := iavlstore.UnsafeNewStore(tree, 10, 10)
+	store := iavlstore.UnsafeNewStore(tree)
 	_ = mngr.GetStoreCache(sKey, store)
 
 	require.Equal(t, store, mngr.Unwrap(sKey))
@@ -48,7 +48,7 @@ func TestStoreCache(t *testing.T) {
 	sKey := types.NewKVStoreKey("test")
 	tree, err := iavl.NewMutableTree(db, 100)
 	require.NoError(t, err)
-	store := iavlstore.UnsafeNewStore(tree, 10, 10)
+	store := iavlstore.UnsafeNewStore(tree)
 	kvStore := mngr.GetStoreCache(sKey, store)
 
 	for i := uint(0); i < cache.DefaultCommitKVStoreCacheSize*2; i++ {

store/iavl/store.go

@@ -1,10 +1,10 @@
 package iavl
 
 import (
+	"fmt"
 	"io"
 	"sync"
 
-	"github.com/pkg/errors"
 	"github.com/tendermint/iavl"
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/crypto/merkle"
@@ -31,25 +31,22 @@ var (
 // Store Implements types.KVStore and CommitKVStore.
 type Store struct {
 	tree Tree
-
-	// How many old versions we hold onto.
-	// A value of 0 means keep no recent states.
-	numRecent int64
-
-	// This is the distance between state-sync waypoint states to be stored.
-	// See https://github.com/tendermint/tendermint/issues/828
-	// A value of 1 means store every state.
-	// A value of 0 means store no waypoints. (node cannot assist in state-sync)
-	// By default this value should be set the same across all nodes,
-	// so that nodes can know the waypoints their peers store.
-	storeEvery int64
 }
 
 // LoadStore returns an IAVL Store as a CommitKVStore. Internally it will load the
 // store's version (id) from the provided DB. An error is returned if the version
 // fails to load.
 func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyLoading bool) (types.CommitKVStore, error) {
-	tree, err := iavl.NewMutableTree(db, defaultIAVLCacheSize)
+	var iavlOpts *iavl.Options
+	iavlOpts = iavl.PruningOptions(pruning.KeepEvery(), pruning.KeepRecent())
+	fmt.Printf("iavlOpts: %v\n", iavlOpts)
+	tree, err := iavl.NewMutableTreeWithOpts(
+		db,
+		dbm.NewMemDB(),
+		defaultIAVLCacheSize,
+		iavlOpts,
+	)
+	fmt.Printf("IAVL ERR: %v\n", err)
 	if err != nil {
 		return nil, err
 	}
@@ -64,21 +61,15 @@ func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyL
 		return nil, err
 	}
 
-	iavl := UnsafeNewStore(tree, int64(0), int64(0))
-	iavl.SetPruning(pruning)
-
-	return iavl, nil
+	return &Store{tree: tree}, nil
 }
 
-// UnsafeNewStore returns a reference to a new IAVL Store.
+// UnsafeNewStore returns a reference to a new IAVL Store with a given mutable
+// IAVL tree reference.
 //
 // CONTRACT: The IAVL tree should be fully loaded.
-func UnsafeNewStore(tree *iavl.MutableTree, numRecent int64, storeEvery int64) *Store {
-	return &Store{
-		tree:       tree,
-		numRecent:  numRecent,
-		storeEvery: storeEvery,
-	}
+func UnsafeNewStore(tree *iavl.MutableTree) *Store {
+	return &Store{tree: tree}
 }
 
 // GetImmutable returns a reference to a new store backed by an immutable IAVL
@@ -96,11 +87,7 @@ func (st *Store) GetImmutable(version int64) (*Store, error) {
 		return nil, err
 	}
 
-	return &Store{
-		tree:       &immutableTree{iTree},
-		numRecent:  0,
-		storeEvery: 0,
-	}, nil
+	return &Store{tree: &immutableTree{iTree}}, nil
 }
 
 // Implements Committer.
@@ -112,18 +99,6 @@ func (st *Store) Commit() types.CommitID {
 		panic(err)
 	}
 
-	// Release an old version of history, if not a sync waypoint.
-	previous := version - 1
-	if st.numRecent < previous {
-		toRelease := previous - st.numRecent
-		if st.storeEvery == 0 || toRelease%st.storeEvery != 0 {
-			err := st.tree.DeleteVersion(toRelease)
-			if errCause := errors.Cause(err); errCause != nil && errCause != iavl.ErrVersionDoesNotExist {
-				panic(err)
-			}
-		}
-	}
-
 	return types.CommitID{
 		Version: version,
 		Hash:    hash,
@@ -138,10 +113,10 @@ func (st *Store) LastCommitID() types.CommitID {
 	}
 }
 
-// Implements Committer.
-func (st *Store) SetPruning(opt types.PruningOptions) {
-	st.numRecent = opt.KeepRecent()
-	st.storeEvery = opt.KeepEvery()
+// SetPruning panics as pruning options should be provided at initialization
+// since IAVl accepts pruning options directly.
+func (st *Store) SetPruning(_ types.PruningOptions) {
+	panic("cannot set pruning options on an initialized IAVL store")
 }
 
 // VersionExists returns whether or not a given version is stored.

store/iavl/store_test.go

@@ -58,7 +58,7 @@ func newAlohaTree(t *testing.T, db dbm.DB) (*iavl.MutableTree, types.CommitID) {
 func TestGetImmutable(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, cID := newAlohaTree(t, db)
-	store := UnsafeNewStore(tree, 10, 10)
+	store := UnsafeNewStore(tree)
 
 	require.True(t, tree.Set([]byte("hello"), []byte("adios")))
 	hash, ver, err := tree.SaveVersion()
@@ -88,7 +88,7 @@ func TestGetImmutable(t *testing.T) {
 func TestTestGetImmutableIterator(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, cID := newAlohaTree(t, db)
-	store := UnsafeNewStore(tree, 10, 10)
+	store := UnsafeNewStore(tree)
 
 	newStore, err := store.GetImmutable(cID.Version)
 	require.NoError(t, err)
@@ -111,7 +111,7 @@ func TestTestGetImmutableIterator(t *testing.T) {
 func TestIAVLStoreGetSetHasDelete(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, _ := newAlohaTree(t, db)
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := UnsafeNewStore(tree)
 
 	key := "hello"
 
@@ -136,14 +136,14 @@ func TestIAVLStoreGetSetHasDelete(t *testing.T) {
 func TestIAVLStoreNoNilSet(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, _ := newAlohaTree(t, db)
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := UnsafeNewStore(tree)
 	require.Panics(t, func() { iavlStore.Set([]byte("key"), nil) }, "setting a nil value should panic")
 }
 
 func TestIAVLIterator(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, _ := newAlohaTree(t, db)
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := UnsafeNewStore(tree)
 	iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz"))
 	expected := []string{"aloha", "hello"}
 	var i int
@@ -219,7 +219,7 @@ func TestIAVLReverseIterator(t *testing.T) {
 	tree, err := iavl.NewMutableTree(db, cacheSize)
 	require.NoError(t, err)
 
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := UnsafeNewStore(tree)
 
 	iavlStore.Set([]byte{0x00}, []byte("0"))
 	iavlStore.Set([]byte{0x00, 0x00}, []byte("0 0"))
@@ -252,7 +252,7 @@ func TestIAVLPrefixIterator(t *testing.T) {
 	tree, err := iavl.NewMutableTree(db, cacheSize)
 	require.NoError(t, err)
 
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := UnsafeNewStore(tree)
 
 	iavlStore.Set([]byte("test1"), []byte("test1"))
 	iavlStore.Set([]byte("test2"), []byte("test2"))
@@ -316,7 +316,7 @@ func TestIAVLReversePrefixIterator(t *testing.T) {
 	tree, err := iavl.NewMutableTree(db, cacheSize)
 	require.NoError(t, err)
 
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := UnsafeNewStore(tree)
 
 	iavlStore.Set([]byte("test1"), []byte("test1"))
 	iavlStore.Set([]byte("test2"), []byte("test2"))
@@ -389,16 +389,16 @@ func TestIAVLDefaultPruning(t *testing.T) {
 		{[]int64{1, 2, 3}, []int64{}},
 		{[]int64{1, 2, 3, 4}, []int64{}},
 		{[]int64{1, 2, 3, 4, 5}, []int64{}},
-		{[]int64{1, 2, 3, 4, 5, 6}, []int64{}},
-		{[]int64{2, 3, 4, 5, 6, 7}, []int64{1}},
+		{[]int64{2, 3, 4, 5, 6}, []int64{1}},
+		{[]int64{3, 4, 5, 6, 7}, []int64{1, 2}},
 		{[]int64{3, 4, 5, 6, 7, 8}, []int64{1, 2}},
-		{[]int64{3, 4, 5, 6, 7, 8, 9}, []int64{1, 2}},
-		{[]int64{3, 5, 6, 7, 8, 9, 10}, []int64{1, 2, 4}},
+		{[]int64{3, 5, 6, 7, 8, 9}, []int64{1, 2, 4}},
+		{[]int64{3, 6, 7, 8, 9, 10}, []int64{1, 2, 4, 5}},
 		{[]int64{3, 6, 7, 8, 9, 10, 11}, []int64{1, 2, 4, 5}},
-		{[]int64{3, 6, 7, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5}},
-		{[]int64{3, 6, 8, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7}},
+		{[]int64{3, 6, 8, 9, 10, 11, 12}, []int64{1, 2, 4, 5, 7}},
+		{[]int64{3, 6, 9, 10, 11, 12, 13}, []int64{1, 2, 4, 5, 7, 8}},
 		{[]int64{3, 6, 9, 10, 11, 12, 13, 14}, []int64{1, 2, 4, 5, 7, 8}},
-		{[]int64{3, 6, 9, 10, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8}},
+		{[]int64{3, 6, 9, 11, 12, 13, 14, 15}, []int64{1, 2, 4, 5, 7, 8, 10}},
 	}
 	testPruning(t, int64(5), int64(3), states)
 }
@@ -411,18 +411,18 @@ func TestIAVLAlternativePruning(t *testing.T) {
 		{[]int64{1}, []int64{}},
 		{[]int64{1, 2}, []int64{}},
 		{[]int64{1, 2, 3}, []int64{}},
-		{[]int64{1, 2, 3, 4}, []int64{}},
-		{[]int64{2, 3, 4, 5}, []int64{1}},
-		{[]int64{3, 4, 5, 6}, []int64{1, 2}},
-		{[]int64{4, 5, 6, 7}, []int64{1, 2, 3}},
+		{[]int64{2, 3, 4}, []int64{1}},
+		{[]int64{3, 4, 5}, []int64{1, 2}},
+		{[]int64{4, 5, 6}, []int64{1, 2, 3}},
+		{[]int64{5, 6, 7}, []int64{1, 2, 3, 4}},
 		{[]int64{5, 6, 7, 8}, []int64{1, 2, 3, 4}},
-		{[]int64{5, 6, 7, 8, 9}, []int64{1, 2, 3, 4}},
-		{[]int64{5, 7, 8, 9, 10}, []int64{1, 2, 3, 4, 6}},
-		{[]int64{5, 8, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7}},
-		{[]int64{5, 9, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8}},
+		{[]int64{5, 7, 8, 9}, []int64{1, 2, 3, 4, 6}},
+		{[]int64{5, 8, 9, 10}, []int64{1, 2, 3, 4, 6, 7}},
+		{[]int64{5, 9, 10, 11}, []int64{1, 2, 3, 4, 6, 7, 8}},
+		{[]int64{5, 10, 11, 12}, []int64{1, 2, 3, 4, 6, 7, 8, 9}},
 		{[]int64{5, 10, 11, 12, 13}, []int64{1, 2, 3, 4, 6, 7, 8, 9}},
-		{[]int64{5, 10, 11, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9}},
-		{[]int64{5, 10, 12, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}},
+		{[]int64{5, 10, 12, 13, 14}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11}},
+		{[]int64{5, 10, 13, 14, 15}, []int64{1, 2, 3, 4, 6, 7, 8, 9, 11, 12}},
 	}
 	testPruning(t, int64(3), int64(5), states)
 }
@@ -434,10 +434,14 @@ type pruneState struct {
 
 func testPruning(t *testing.T, numRecent int64, storeEvery int64, states []pruneState) {
 	db := dbm.NewMemDB()
-	tree, err := iavl.NewMutableTree(db, cacheSize)
-	require.NoError(t, err)
-
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	var iavlOpts *iavl.Options
+	if numRecent == 0 && storeEvery == 0 {
+		iavlOpts = iavl.DefaultOptions()
+	} else {
+		iavlOpts = iavl.PruningOptions(storeEvery, numRecent)
+	}
+	tree, _ := iavl.NewMutableTreeWithOpts(db, dbm.NewMemDB(), cacheSize, iavlOpts)
+	iavlStore := UnsafeNewStore(tree)
 
 	for step, state := range states {
 		for _, ver := range state.stored {
@@ -461,7 +465,7 @@ func TestIAVLNoPrune(t *testing.T) {
 	tree, err := iavl.NewMutableTree(db, cacheSize)
 	require.NoError(t, err)
 
-	iavlStore := UnsafeNewStore(tree, numRecent, int64(1))
+	iavlStore := UnsafeNewStore(tree)
 	nextVersion(iavlStore)
 
 	for i := 1; i < 100; i++ {
@@ -477,10 +481,12 @@ func TestIAVLNoPrune(t *testing.T) {
 
 func TestIAVLPruneEverything(t *testing.T) {
 	db := dbm.NewMemDB()
-	tree, err := iavl.NewMutableTree(db, cacheSize)
+	var iavlOpts *iavl.Options
+	iavlOpts = iavl.PruningOptions(0, 1) // only store latest version in memory
+	tree, err := iavl.NewMutableTreeWithOpts(db, dbm.NewMemDB(), cacheSize, iavlOpts)
 	require.NoError(t, err)
 
-	iavlStore := UnsafeNewStore(tree, int64(0), int64(0))
+	iavlStore := UnsafeNewStore(tree)
 	nextVersion(iavlStore)
 
 	for i := 1; i < 100; i++ {
@@ -503,7 +509,7 @@ func TestIAVLStoreQuery(t *testing.T) {
 	tree, err := iavl.NewMutableTree(db, cacheSize)
 	require.NoError(t, err)
 
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := UnsafeNewStore(tree)
 
 	k1, v1 := []byte("key1"), []byte("val1")
 	k2, v2 := []byte("key2"), []byte("val2")
@@ -602,7 +608,7 @@ func BenchmarkIAVLIteratorNext(b *testing.B) {
 		tree.Set(key, value)
 	}
 
-	iavlStore := UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := UnsafeNewStore(tree)
 	iterators := make([]types.Iterator, b.N/treeSize)
 
 	for i := 0; i < len(iterators); i++ {

store/prefix/store_test.go

@@ -90,7 +90,7 @@ func TestIAVLStorePrefix(t *testing.T) {
 	db := dbm.NewMemDB()
 	tree, err := tiavl.NewMutableTree(db, cacheSize)
 	require.NoError(t, err)
-	iavlStore := iavl.UnsafeNewStore(tree, numRecent, storeEvery)
+	iavlStore := iavl.UnsafeNewStore(tree)
 
 	testPrefixStore(t, iavlStore, []byte("test"))
 }

store/rootmulti/proof_test.go

@@ -61,7 +61,8 @@ func TestVerifyMultiStoreQueryProof(t *testing.T) {
 	iavlStoreKey := types.NewKVStoreKey("iavlStoreKey")
 
 	store.MountStoreWithDB(iavlStoreKey, types.StoreTypeIAVL, nil)
-	store.LoadVersion(0)
+	serr := store.LoadVersion(0)
+	require.Nil(t, serr)
 
 	iavlStore := store.GetCommitStore(iavlStoreKey).(*iavl.Store)
 	iavlStore.Set([]byte("MYKEY"), []byte("MYVALUE"))