Introduce Pruning to IAVL

.golangci.yml

@@ -61,4 +61,4 @@ linters-settings:
 #     enabled-tags:
 #       - performance
 #       - style
-#       - experimental
+#       - experimental

CHANGELOG_PENDING.md

@@ -7,6 +7,8 @@ Special thanks to external contributors on this release:
 
 ### BREAKING CHANGES
 
+- [/#158] NodeDB constructor must provide `keepRecent` and `keepEvery` fields to define PruningStrategy. All Save functionality must specify whether they should flushToDisk as well using `flushToDisk` boolean argument. All Delete functionality must specify whether object should be deleted from memory only using the `memOnly` boolean argument.
+
 ### IMPROVEMENTS
 
 ### Bug Fix

Makefile

@@ -51,6 +51,9 @@ fullbench:
 		go test $(LDFLAGS) -bench=Mem . && \
 		go test $(LDFLAGS) -timeout=60m -bench=LevelDB .
 
+benchprune:
+	cd benchmarks && \
+		go test -bench=PruningStrategies -timeout=24h
 
 # note that this just profiles the in-memory version, not persistence
 profile:

basic_test.go

@@ -13,7 +13,8 @@ import (
 )
 
 func TestBasic(t *testing.T) {
-	tree := NewMutableTree(db.NewMemDB(), 0)
+	tree, err := getTestTree(0)
+	require.NoError(t, err)
 	up := tree.Set([]byte("1"), []byte("one"))
 	if up {
 		t.Error("Did not expect an update (should have been create)")
@@ -186,12 +187,11 @@ func TestUnit(t *testing.T) {
 }
 
 func TestRemove(t *testing.T) {
-	size := 10000
 	keyLen, dataLen := 16, 40
 
-	d := db.NewDB("test", "memdb", "")
-	defer d.Close()
-	t1 := NewMutableTree(d, size)
+	size := 10000
+	t1, err := getTestTree(size)
+	require.NoError(t, err)
 
 	// insert a bunch of random nodes
 	keys := make([][]byte, size)
@@ -221,7 +221,8 @@ func TestIntegration(t *testing.T) {
 	}
 
 	records := make([]*record, 400)
-	tree := NewMutableTree(db.NewMemDB(), 0)
+	tree, err := getTestTree(0)
+	require.NoError(t, err)
 
 	randomRecord := func() *record {
 		return &record{randstr(20), randstr(20)}
@@ -303,7 +304,8 @@ func TestIterateRange(t *testing.T) {
 	}
 	sort.Strings(keys)
 
-	tree := NewMutableTree(db.NewMemDB(), 0)
+	tree, err := getTestTree(0)
+	require.NoError(t, err)
 
 	// insert all the data
 	for _, r := range records {
@@ -373,14 +375,16 @@ func TestPersistence(t *testing.T) {
 	}
 
 	// Construct some tree and save it
-	t1 := NewMutableTree(db, 0)
+	t1, err := NewMutableTree(db, 0)
+	require.NoError(t, err)
 	for key, value := range records {
 		t1.Set([]byte(key), []byte(value))
 	}
 	t1.SaveVersion()
 
 	// Load a tree
-	t2 := NewMutableTree(db, 0)
+	t2, err := NewMutableTree(db, 0)
+	require.NoError(t, err)
 	t2.Load()
 	for key, value := range records {
 		_, t2value := t2.Get([]byte(key))
@@ -393,8 +397,8 @@ func TestPersistence(t *testing.T) {
 func TestProof(t *testing.T) {
 
 	// Construct some random tree
-	db := db.NewMemDB()
-	tree := NewMutableTree(db, 100)
+	tree, err := getTestTree(100)
+	require.NoError(t, err)
 	for i := 0; i < 10; i++ {
 		key, value := randstr(20), randstr(20)
 		tree.Set([]byte(key), []byte(value))
@@ -423,7 +427,8 @@ func TestProof(t *testing.T) {
 
 func TestTreeProof(t *testing.T) {
 	db := db.NewMemDB()
-	tree := NewMutableTree(db, 100)
+	tree, err := NewMutableTree(db, 100)
+	require.NoError(t, err)
 	assert.Equal(t, tree.Hash(), []byte(nil))
 
 	// should get false for proof with nil root

benchmarks/bench_test.go

@@ -3,6 +3,7 @@ package benchmarks
 
 import (
 	"fmt"
+	"github.com/stretchr/testify/require"
 	"math/rand"
 	"os"
 	"runtime"
@@ -12,8 +13,6 @@ import (
 	db "github.com/tendermint/tm-db"
 )
 
-const historySize = 20
-
 func randBytes(length int) []byte {
 	key := make([]byte, length)
 	// math.rand.Read always returns err=nil
@@ -23,8 +22,9 @@ func randBytes(length int) []byte {
 	return key
 }
 
-func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) {
-	t := iavl.NewMutableTree(db, size)
+func prepareTree(b *testing.B, snapdb db.DB, memdb db.DB, keepEvery int64, keepRecent int64, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) {
+	t, err := iavl.NewMutableTreeWithOpts(snapdb, memdb, size, iavl.PruningOptions(keepEvery, keepRecent))
+	require.NoError(b, err)
 	keys := make([][]byte, size)
 
 	for i := 0; i < size; i++ {
@@ -37,19 +37,15 @@ func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.Mutab
 	return t, keys
 }
 
-// commit tree saves a new version and deletes and old one...
+// commit tree saves a new version according to pruning strategy passed into IAVL
 func commitTree(b *testing.B, t *iavl.MutableTree) {
 	t.Hash()
-	_, version, err := t.SaveVersion()
+
+	_, _, err := t.SaveVersion() //this will flush for us every so often
+
 	if err != nil {
 		b.Errorf("Can't save: %v", err)
 	}
-	if version > historySize {
-		err = t.DeleteVersion(version - historySize)
-		if err != nil {
-			b.Errorf("Can't delete: %v", err)
-		}
-	}
 }
 
 func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) {
@@ -220,28 +216,38 @@ func BenchmarkLevelDBLargeData(b *testing.B) {
 
 func runBenchmarks(b *testing.B, benchmarks []benchmark) {
 	fmt.Printf("%s\n", iavl.GetVersionInfo())
-	for _, bb := range benchmarks {
-		prefix := fmt.Sprintf("%s-%d-%d-%d-%d", bb.dbType, bb.initSize,
-			bb.blockSize, bb.keyLen, bb.dataLen)
-
-		// prepare a dir for the db and cleanup afterwards
-		dirName := fmt.Sprintf("./%s-db", prefix)
-		defer func() {
-			err := os.RemoveAll(dirName)
-			if err != nil {
-				b.Errorf("%+v\n", err)
+	pruningStrategies := []pruningstrat{
+		{1, 0},       // default pruning strategy
+		{0, 1},       // keep single recent version
+		{100, 5},     // simple pruning
+		{1000, 10},   // average pruning
+		{1000, 1},    // extreme pruning
+		{10000, 100}, // SDK pruning
+	}
+	for _, ps := range pruningStrategies {
+		for _, bb := range benchmarks {
+			prefix := fmt.Sprintf("%s-%d-%d-%d-%d-%d-%d", bb.dbType, ps.keepEvery, ps.keepRecent,
+				bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen)
+
+			// prepare a dir for the db and cleanup afterwards
+			dirName := fmt.Sprintf("./%s-db", prefix)
+			defer func() {
+				err := os.RemoveAll(dirName)
+				if err != nil {
+					b.Errorf("%+v\n", err)
+				}
+			}()
+
+			// note that "" leads to nil backing db!
+			var d db.DB
+			if bb.dbType != "nodb" {
+				d = db.NewDB("test", bb.dbType, dirName)
+				defer d.Close()
 			}
-		}()
-
-		// note that "" leads to nil backing db!
-		var d db.DB
-		if bb.dbType != "nodb" {
-			d = db.NewDB("test", bb.dbType, dirName)
-			defer d.Close()
+			b.Run(prefix, func(sub *testing.B) {
+				runSuite(sub, d, ps.keepEvery, ps.keepRecent, bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen)
+			})
 		}
-		b.Run(prefix, func(sub *testing.B) {
-			runSuite(sub, d, bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen)
-		})
 	}
 }
 
@@ -254,12 +260,12 @@ func memUseMB() float64 {
 	return mb
 }
 
-func runSuite(b *testing.B, d db.DB, initSize, blockSize, keyLen, dataLen int) {
+func runSuite(b *testing.B, d db.DB, keepEvery int64, keepRecent int64, initSize, blockSize, keyLen, dataLen int) {
 	// measure mem usage
 	runtime.GC()
 	init := memUseMB()
 
-	t, keys := prepareTree(b, d, initSize, keyLen, dataLen)
+	t, keys := prepareTree(b, d, db.NewMemDB(), keepEvery, keepRecent, initSize, keyLen, dataLen)
 	used := memUseMB() - init
 	fmt.Printf("Init Tree took %0.2f MB\n", used)
 

benchmarks/prune_test.go

@@ -0,0 +1,98 @@
+package benchmarks
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"runtime"
+	"testing"
+
+	db "github.com/tendermint/tm-db"
+)
+
+type pruningstrat struct {
+	keepEvery, keepRecent int64
+}
+
+// To test effect of pruning strategy, we must measure time to execute many blocks
+// Execute 30000 blocks with the given IAVL tree's pruning strategy
+func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int64, keyLen, dataLen int) {
+	// prepare a dir for the db and cleanup afterwards
+	dirName := fmt.Sprintf("./%s-db", prefix)
+	defer func() {
+		err := os.RemoveAll(dirName)
+		if err != nil {
+			b.Errorf("%+v\n", err)
+		}
+	}()
+
+	runtime.GC()
+
+	// always initialize tree with goleveldb as snapshotDB and memDB as recentDB
+	snapDB := db.NewDB("test", "goleveldb", dirName)
+	defer snapDB.Close()
+
+	var mem runtime.MemStats
+	runtime.ReadMemStats(&mem)
+	memSize := mem.Alloc
+	//maxVersion := 0
+	var keys [][]byte
+	for i := 0; i < 100; i++ {
+		keys = append(keys, randBytes(keyLen))
+	}
+
+	// reset timer after initialization logic
+	b.ResetTimer()
+	t, _ := prepareTree(b, snapDB, db.NewMemDB(), keepEvery, keepRecent, 5, keyLen, dataLen)
+
+	// create 30000 versions
+	for i := 0; i < 5000; i++ {
+		// set 5 keys per version
+		for j := 0; j < 5; j++ {
+			index := rand.Int63n(100)
+			t.Set(keys[index], randBytes(dataLen))
+		}
+		_, _, err := t.SaveVersion()
+		if err != nil {
+			b.Errorf("Can't save version %d: %v", i, err)
+		}
+
+		// Pause timer to garbage-collect and remeasure memory usage
+		b.StopTimer()
+		runtime.GC()
+		runtime.ReadMemStats(&mem)
+		// update memSize if it has increased after saveVersion
+		if memSize < mem.Alloc {
+			memSize = mem.Alloc
+			//maxVersion = i
+		}
+		b.StartTimer()
+	}
+
+	//fmt.Printf("Maxmimum Memory usage was %0.2f MB at height %d\n", float64(memSize)/1000000, maxVersion)
+	b.StopTimer()
+}
+
+func BenchmarkPruningStrategies(b *testing.B) {
+	ps := []pruningstrat{
+		{1, 0}, // default pruning strategy
+		{1, 1},
+		{0, 1}, // keep single recent version
+		{100, 1},
+		{100, 5}, // simple pruning
+		{5, 1},
+		{5, 2},
+		{10, 2},
+		{1000, 10},   // average pruning
+		{1000, 1},    // extreme pruning
+		{10000, 100}, // SDK pruning
+	}
+	for _, ps := range ps {
+		ps := ps
+		prefix := fmt.Sprintf("PruningStrategy{%d-%d}-KeyLen:%d-DataLen:%d", ps.keepEvery, ps.keepRecent, 16, 40)
+
+		b.Run(prefix, func(sub *testing.B) {
+			runBlockChain(sub, prefix, ps.keepEvery, ps.keepRecent, 16, 40)
+		})
+	}
+}

benchmarks/results/158-pruning/Adityas-MBP-pruning-results.txt

@@ -0,0 +1,9 @@
+goos: darwin
+goarch: amd64
+pkg: github.com/tendermint/iavl/benchmarks
+BenchmarkPruningStrategies/PruningStrategy{1-0}-KeyLen:16-DataLen:40-8         	       1	2837806322 ns/op
+BenchmarkPruningStrategies/PruningStrategy{0-1}-KeyLen:16-DataLen:40-8         	       1	1124373981 ns/op
+BenchmarkPruningStrategies/PruningStrategy{100-1}-KeyLen:16-DataLen:40-8       	       1	1255040658 ns/op
+BenchmarkPruningStrategies/PruningStrategy{100-5}-KeyLen:16-DataLen:40-8       	       1	1459752743 ns/op
+PASS
+ok  	github.com/tendermint/iavl/benchmarks	12.375s