Introduce Pruning to IAVL

CHANGELOG_PENDING.md

@@ -6,4 +6,10 @@ Special thanks to external contributors on this release:
 
 ### BREAKING CHANGES
 
+- [/#158] NodeDB constructor must provide `keepRecent` and `keepEvery` fields to define PruningStrategy. All Save functionality must specify whether they should flushToDisk as well using `flushToDisk` boolean argument. All Delete functionality must specify whether object should be deleted from memory only using the `memOnly` boolean argument.
+
 ### IMPROVEMENTS
+
+- [/#46](https://github.com/tendermint/iavl/issues/46) Removal of all instances of cmn from tendermint in Ival repo
+
+- [/#158] Add constructor to IAVL that can define a pruning strategy. Only persist to disk snapshot versions and keep a specified number of versions in memDB. This greatly reduces IO load of IAVL.

Makefile

@@ -49,6 +49,9 @@ fullbench:
 		go test $(LDFLAGS) -bench=Mem . && \
 		go test $(LDFLAGS) -timeout=60m -bench=LevelDB .
 
+benchprune:
+	cd benchmarks && \
+		go test -bench=PruningStrategies -timeout=24h
 
 # note that this just profiles the in-memory version, not persistence
 profile:

PRUNING.md

@@ -0,0 +1,38 @@
+# Pruning
+
+Setting Pruning fields in the IAVL tree can optimize performance by only writing versions to disk if they are meant to be persisted indefinitely. Versions that are known to be deleted eventually are temporarily held in memory until they are ready to be pruned. This greatly reduces the I/O load of IAVL.
+
+We can set custom pruning fields in IAVL using: `NewMutableTreePruningOpts`
+
+
+## Current design
+
+### NodeDB
+NodeDB has extra fields:
+
+```go
+recentDB    dbm.DB     // Memory node storage.
+recentBatch dbm.Batch  // Batched writing buffer for memDB.
+
+// Pruning fields
+keepEvery  int64n // Saves version to disk periodically
+keepRecent int64  // Saves recent versions in memory
+```
+
+If version is not going to be persisted to disk, the version is simply saved in `recentDB` (typically a `memDB`)
+If version is persisted to disk, the version is written to `recentDB` **and** `snapshotDB` (typically `levelDB`)
+
+#### Orphans:
+
+Save orphan to `memDB` under `o|toVersion|fromVersion`.
+
+If there exists snapshot version `snapVersion` s.t. `fromVersion < snapVersion < toVersion`, save orphan to disk as well under `o|snapVersion|fromVersion`.
+NOTE: in unlikely event, that two snapshot versions exist between `fromVersion` and `toVersion`, we use closest snapshot version that is less than `toVersion`
+
+Can then simply use the old delete algorithm with some minor simplifications/optimizations
+
+### MutableTree
+
+MutableTree can be instantiated with a pruning-aware NodeDB.
+
+When `MutableTree` saves a new Version, it also calls `PruneRecentVersions` on nodeDB which causes oldest version in recentDB (`latestVersion - keepRecent`) to get pruned.

basic_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestBasic(t *testing.T) {
-	tree := NewMutableTree(db.NewMemDB(), 0)
+	tree := getTestTree(0)
 	up := tree.Set([]byte("1"), []byte("one"))
 	if up {
 		t.Error("Did not expect an update (should have been create)")
@@ -185,12 +185,10 @@ func TestUnit(t *testing.T) {
 }
 
 func TestRemove(t *testing.T) {
-	size := 10000
 	keyLen, dataLen := 16, 40
 
-	d := db.NewDB("test", "memdb", "")
-	defer d.Close()
-	t1 := NewMutableTree(d, size)
+	size := 10000
+	t1 := getTestTree(size)
 
 	// insert a bunch of random nodes
 	keys := make([][]byte, size)
@@ -220,7 +218,7 @@ func TestIntegration(t *testing.T) {
 	}
 
 	records := make([]*record, 400)
-	tree := NewMutableTree(db.NewMemDB(), 0)
+	tree := getTestTree(0)
 
 	randomRecord := func() *record {
 		return &record{randstr(20), randstr(20)}
@@ -302,7 +300,7 @@ func TestIterateRange(t *testing.T) {
 	}
 	sort.Strings(keys)
 
-	tree := NewMutableTree(db.NewMemDB(), 0)
+	tree := getTestTree(0)
 
 	// insert all the data
 	for _, r := range records {
@@ -392,8 +390,7 @@ func TestPersistence(t *testing.T) {
 func TestProof(t *testing.T) {
 
 	// Construct some random tree
-	db := db.NewMemDB()
-	tree := NewMutableTree(db, 100)
+	tree := getTestTree(100)
 	for i := 0; i < 10; i++ {
 		key, value := randstr(20), randstr(20)
 		tree.Set([]byte(key), []byte(value))

benchmarks/bench_test.go

@@ -11,8 +11,6 @@ import (
 	db "github.com/tendermint/tm-db"
 )
 
-const historySize = 20
-
 func randBytes(length int) []byte {
 	key := make([]byte, length)
 	// math.rand.Read always returns err=nil
@@ -22,8 +20,8 @@ func randBytes(length int) []byte {
 	return key
 }
 
-func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) {
-	t := iavl.NewMutableTree(db, size)
+func prepareTree(b *testing.B, snapdb db.DB, memdb db.DB, keepEvery int64, keepRecent int64, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) {
+	t := iavl.NewMutableTreeWithOpts(snapdb, memdb, size, iavl.PruningOptions(keepEvery, keepRecent))
 	keys := make([][]byte, size)
 
 	for i := 0; i < size; i++ {
@@ -36,19 +34,15 @@ func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.Mutab
 	return t, keys
 }
 
-// commit tree saves a new version and deletes and old one...
+// commit tree saves a new version according to pruning strategy passed into IAVL
 func commitTree(b *testing.B, t *iavl.MutableTree) {
 	t.Hash()
-	_, version, err := t.SaveVersion()
+
+	_, _, err := t.SaveVersion() //this will flush for us every so often
+
 	if err != nil {
 		b.Errorf("Can't save: %v", err)
 	}
-	if version > historySize {
-		err = t.DeleteVersion(version - historySize)
-		if err != nil {
-			b.Errorf("Can't delete: %v", err)
-		}
-	}
 }
 
 func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) {
@@ -219,28 +213,38 @@ func BenchmarkLevelDBLargeData(b *testing.B) {
 
 func runBenchmarks(b *testing.B, benchmarks []benchmark) {
 	fmt.Printf("%s\n", iavl.GetVersionInfo())
-	for _, bb := range benchmarks {
-		prefix := fmt.Sprintf("%s-%d-%d-%d-%d", bb.dbType, bb.initSize,
-			bb.blockSize, bb.keyLen, bb.dataLen)
-
-		// prepare a dir for the db and cleanup afterwards
-		dirName := fmt.Sprintf("./%s-db", prefix)
-		defer func() {
-			err := os.RemoveAll(dirName)
-			if err != nil {
-				b.Errorf("%+v\n", err)
+	pruningStrategies := []pruningstrat{
+		{1, 0},       // default pruning strategy
+		{0, 1},       // keep single recent version
+		{100, 5},     // simple pruning
+		{1000, 10},   // average pruning
+		{1000, 1},    // extreme pruning
+		{10000, 100}, // SDK pruning
+	}
+	for _, ps := range pruningStrategies {
+		for _, bb := range benchmarks {
+			prefix := fmt.Sprintf("%s-%d-%d-%d-%d-%d-%d", bb.dbType, ps.keepEvery, ps.keepRecent,
+				bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen)
+
+			// prepare a dir for the db and cleanup afterwards
+			dirName := fmt.Sprintf("./%s-db", prefix)
+			defer func() {
+				err := os.RemoveAll(dirName)
+				if err != nil {
+					b.Errorf("%+v\n", err)
+				}
+			}()
+
+			// note that "" leads to nil backing db!
+			var d db.DB
+			if bb.dbType != "nodb" {
+				d = db.NewDB("test", bb.dbType, dirName)
+				defer d.Close()
 			}
-		}()
-
-		// note that "" leads to nil backing db!
-		var d db.DB
-		if bb.dbType != "nodb" {
-			d = db.NewDB("test", bb.dbType, dirName)
-			defer d.Close()
+			b.Run(prefix, func(sub *testing.B) {
+				runSuite(sub, d, ps.keepEvery, ps.keepRecent, bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen)
+			})
 		}
-		b.Run(prefix, func(sub *testing.B) {
-			runSuite(sub, d, bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen)
-		})
 	}
 }
 
@@ -253,12 +257,12 @@ func memUseMB() float64 {
 	return mb
 }
 
-func runSuite(b *testing.B, d db.DB, initSize, blockSize, keyLen, dataLen int) {
+func runSuite(b *testing.B, d db.DB, keepEvery int64, keepRecent int64, initSize, blockSize, keyLen, dataLen int) {
 	// measure mem usage
 	runtime.GC()
 	init := memUseMB()
 
-	t, keys := prepareTree(b, d, initSize, keyLen, dataLen)
+	t, keys := prepareTree(b, d, db.NewMemDB(), keepEvery, keepRecent, initSize, keyLen, dataLen)
 	used := memUseMB() - init
 	fmt.Printf("Init Tree took %0.2f MB\n", used)
 

benchmarks/prune_test.go

@@ -0,0 +1,98 @@
+package benchmarks
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"runtime"
+	"testing"
+
+	db "github.com/tendermint/tm-db"
+)
+
+type pruningstrat struct {
+	keepEvery, keepRecent int64
+}
+
+// To test effect of pruning strategy, we must measure time to execute many blocks
+// Execute 30000 blocks with the given IAVL tree's pruning strategy
+func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int64, keyLen, dataLen int) {
+	// prepare a dir for the db and cleanup afterwards
+	dirName := fmt.Sprintf("./%s-db", prefix)
+	defer func() {
+		err := os.RemoveAll(dirName)
+		if err != nil {
+			b.Errorf("%+v\n", err)
+		}
+	}()
+
+	runtime.GC()
+
+	// always initialize tree with goleveldb as snapshotDB and memDB as recentDB
+	snapDB := db.NewDB("test", "goleveldb", dirName)
+	defer snapDB.Close()
+
+	// var mem runtime.MemStats
+	// runtime.ReadMemStats(&mem)
+	// memSize := mem.Alloc
+	// maxVersion := 0
+	var keys [][]byte
+	for i := 0; i < 100; i++ {
+		keys = append(keys, randBytes(keyLen))
+	}
+
+	// reset timer after initialization logic
+	b.ResetTimer()
+	t, _ := prepareTree(b, snapDB, db.NewMemDB(), keepEvery, keepRecent, 5, keyLen, dataLen)
+
+	// create 30000 versions
+	for i := 0; i < 5000; i++ {
+		// set 5 keys per version
+		for j := 0; j < 5; j++ {
+			index := rand.Int63n(100)
+			t.Set(keys[index], randBytes(dataLen))
+		}
+		_, _, err := t.SaveVersion()
+		if err != nil {
+			b.Errorf("Can't save version %d: %v", i, err)
+		}
+		// // Pause timer to garbage-collect and remeasure memory usage
+		// b.StopTimer()
+		// runtime.GC()
+		// runtime.ReadMemStats(&mem)
+		// // update memSize if it has increased after saveVersion
+		// if memSize < mem.Alloc {
+		// 	memSize = mem.Alloc
+		// 	maxVersion = i
+		// }
+		// b.StartTimer()
+		b.StopTimer()
+		runtime.GC()
+		b.StartTimer()
+	}
+	//fmt.Printf("Maxmimum Memory usage was %0.2f MB at height %d\n", float64(memSize)/1000000, maxVersion)
+	b.StopTimer()
+}
+
+func BenchmarkPruningStrategies(b *testing.B) {
+	ps := []pruningstrat{
+		{1, 0}, // default pruning strategy
+		//{1, 1},
+		{0, 1}, // keep single recent version
+		{100, 1},
+		{100, 5}, // simple pruning
+		{5, 1},
+		{5, 2},
+		{10, 2},
+		// {1000, 10},   // average pruning
+		// {1000, 1},    // extreme pruning
+		// {10000, 100}, // SDK pruning
+	}
+	for _, ps := range ps {
+		prefix := fmt.Sprintf("PruningStrategy{%d-%d}-KeyLen:%d-DataLen:%d", ps.keepEvery, ps.keepRecent, 16, 40)
+
+		b.Run(prefix, func(sub *testing.B) {
+			runBlockChain(sub, prefix, ps.keepEvery, ps.keepRecent, 16, 40)
+		})
+	}
+}

benchmarks/results/158-pruning/Adityas-MBP-pruning-results.txt

@@ -0,0 +1,9 @@
+goos: darwin
+goarch: amd64
+pkg: github.com/tendermint/iavl/benchmarks
+BenchmarkPruningStrategies/PruningStrategy{1-0}-KeyLen:16-DataLen:40-8         	       1	2837806322 ns/op
+BenchmarkPruningStrategies/PruningStrategy{0-1}-KeyLen:16-DataLen:40-8         	       1	1124373981 ns/op
+BenchmarkPruningStrategies/PruningStrategy{100-1}-KeyLen:16-DataLen:40-8       	       1	1255040658 ns/op
+BenchmarkPruningStrategies/PruningStrategy{100-5}-KeyLen:16-DataLen:40-8       	       1	1459752743 ns/op
+PASS
+ok  	github.com/tendermint/iavl/benchmarks	12.375s