From bed48227ae948746bcf50bcbc8724ce94ac18451 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:22:30 +0700 Subject: [PATCH 01/57] Memtree (#11) Storing intermidiary IAVL versions in memory and not to disk Motivation: Both Cosmos and Loom Network save an IAVL version per block, then go back and delete these versions. So you have constant churn on the IAVL and underlying Leveldb database. When realistically what you want is to only store every X Blocks. At Berlin Tendermint Conference, Zaki and I surmised a plan where new versions are in memory, while still pointing back to nodes on disk to prevent needing to load entire IAVL into main memory. Loom IAVL tree is around 256gb so this is not feasible otherwise. Usage OLD Code would be like ```go hash, version, err := s.tree.SaveVersion() ``` New Caller code would look like ```go oldVersion := s.Version() var version int64 var hash []byte //Every X versions we should persist to disk if s.flushInterval == 0 || ((oldVersion+1)%s.flushInterval == 0) { if s.flushInterval != 0 { log.Error(fmt.Sprintf("Flushing mem to disk at version %d\n", oldVersion+1)) hash, version, err = s.tree.FlushMemVersionDisk() } else { hash, version, err = s.tree.SaveVersion() } } else { hash, version, err = s.tree.SaveVersionMem() } ``` FlushMemVersionDisk: Flushes the current memory version to disk SaveVersionMem: Saves the current tree to memory instead of disk and gives you back an apphash This is an opt in feature, you have to call new apis to get it. We also have a PR that demonstrates its usage https://github.com/loomnetwork/loomchain/pull/1232/files We are now commiting every 1000 blocks, so we store 1000x less. Also we have signficant improves in IO at least double from not having to Prune old versions of the IAVL Tree --- benchmarks/bench_test.go | 17 +++++++- mutable_tree.go | 53 ++++++++++++++++++++++-- mutable_tree_test.go | 32 +++++++++++++++ node.go | 23 ++++++----- nodedb.go | 89 +++++++++++++++++++++++++++------------- 5 files changed, 170 insertions(+), 44 deletions(-) create mode 100644 mutable_tree_test.go diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 9284a7a2e..d8d415f32 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -37,16 +37,29 @@ func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.Mutab // commit tree saves a new version and deletes and old one... func commitTree(b *testing.B, t *iavl.MutableTree) { t.Hash() - _, version, err := t.SaveVersion() + var err error + var version int64 + + _, version, err = t.SaveVersionMem() //this will flush for us every so often + if err != nil { b.Errorf("Can't save: %v", err) } if version > historySize { - err = t.DeleteVersion(version - historySize) + err = t.DeleteVersionFull(version-historySize, false) if err != nil { b.Errorf("Can't delete: %v", err) } } + + //Lets flush every X blocks + if version%historySize == 0 { + //We don't need to delete all the versions when using mem versions, just flush to disk + _, _, err = t.FlushMemVersionDisk() + if err != nil { + b.Errorf("Can't save: %v", err) + } + } } func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) { diff --git a/mutable_tree.go b/mutable_tree.go index d1bc48927..1f99b24ca 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -16,6 +16,7 @@ type MutableTree struct { *ImmutableTree // The current, working tree. lastSaved *ImmutableTree // The most recently saved tree. orphans map[string]int64 // Nodes removed by changes to working tree. + memVersions map[int64]bool // The previous, saved versions of the tree in mem. versions map[int64]bool // The previous, saved versions of the tree. ndb *nodeDB } @@ -29,6 +30,7 @@ func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { ImmutableTree: head, lastSaved: head.clone(), orphans: map[string]int64{}, + memVersions: map[int64]bool{}, versions: map[int64]bool{}, ndb: ndb, } @@ -364,11 +366,37 @@ func (tree *MutableTree) GetVersioned(key []byte, version int64) ( return -1, nil } +// SaveVersionMem saves a new tree version to disk, based on the current state of +// the tree. Returns the hash and new version number. +func (tree *MutableTree) SaveVersionMem() ([]byte, int64, error) { + return tree.saveVersion(false) +} + +// FlushMemDisk saves a new tree to disk and removes all the versions in memory +func (tree *MutableTree) FlushMemVersionDisk() ([]byte, int64, error) { + x, y, err := tree.saveVersion(true) + tree.ndb.dbMem = dbm.NewMemDB() + tree.memVersions = map[int64]bool{} + tree.ndb.memNodes = map[string]*Node{} + return x, y, err +} + // SaveVersion saves a new tree version to disk, based on the current state of // the tree. Returns the hash and new version number. func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { + return tree.saveVersion(true) +} + +func (tree *MutableTree) saveVersion(flushToDisk bool) ([]byte, int64, error) { version := tree.version + 1 + if flushToDisk { + tree.ndb.batch = tree.ndb.db.NewBatch() + } else { + tree.ndb.batch = tree.ndb.dbMem.NewBatch() + tree.memVersions[version] = true + } + if tree.versions[version] { //version already exists, throw an error if attempting to overwrite // Same hash means idempotent. Return success. @@ -389,13 +417,15 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { // There can still be orphans, for example if the root is the node being // removed. debug("SAVE EMPTY TREE %v\n", version) - tree.ndb.SaveOrphans(version, tree.orphans) + // Assume orphans not needed any more. So don't save any + tree.ndb.SaveOrphans(version, tree.orphans, flushToDisk) tree.ndb.SaveEmptyRoot(version) } else { debug("SAVE TREE %v\n", version) // Save the current tree. - tree.ndb.SaveBranch(tree.root) - tree.ndb.SaveOrphans(version, tree.orphans) + tree.ndb.SaveBranch(tree.root, flushToDisk) + // Assume orphans not needed any more. So don't save any + tree.ndb.SaveOrphans(version, tree.orphans, flushToDisk) tree.ndb.SaveRoot(tree.root, version) } tree.ndb.Commit() @@ -413,6 +443,23 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { // DeleteVersion deletes a tree version from disk. The version can then no // longer be accessed. func (tree *MutableTree) DeleteVersion(version int64) error { + return tree.DeleteVersionFull(version, true) +} + +// DeleteVersionFull deletes a tree version from disk or memory based on the flag. The version can then no +// longer be accessed. +func (tree *MutableTree) DeleteVersionFull(version int64, memDeleteAlso bool) error { + if tree.memVersions[version] { + //sometimes you dont want to bother deleting versions in memory + if !memDeleteAlso { + return nil + } + tree.ndb.batch = tree.ndb.dbMem.NewBatch() + delete(tree.memVersions, version) + } else { + tree.ndb.batch = tree.ndb.db.NewBatch() + } + if version == 0 { return cmn.NewError("version must be greater than 0") } diff --git a/mutable_tree_test.go b/mutable_tree_test.go new file mode 100644 index 000000000..9a1812be0 --- /dev/null +++ b/mutable_tree_test.go @@ -0,0 +1,32 @@ +package iavl + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/db" +) + +func TestDelete(t *testing.T) { + memDb := db.NewMemDB() + tree := NewMutableTree(memDb, 0) + + tree.set([]byte("k1"), []byte("Fred")) + hash, version, err := tree.SaveVersion() + require.NoError(t, err) + _, _, err = tree.SaveVersion() + require.NoError(t, err) + + require.NoError(t, tree.DeleteVersion(version)) + + k1Value, _, err := tree.GetVersionedWithProof([]byte("k1"), version) + require.Nil(t, k1Value) + + key := tree.ndb.rootKey(version) + memDb.Set(key, hash) + tree.versions[version] = true + + k1Value, _, err = tree.GetVersionedWithProof([]byte("k1"), version) + require.Equal(t, 0, bytes.Compare([]byte("Fred"), k1Value)) +} diff --git a/node.go b/node.go index 863751b6a..c039401ba 100644 --- a/node.go +++ b/node.go @@ -15,17 +15,18 @@ import ( // Node represents a node in a Tree. type Node struct { - key []byte - value []byte - version int64 - height int8 - size int64 - hash []byte - leftHash []byte - leftNode *Node - rightHash []byte - rightNode *Node - persisted bool + key []byte + value []byte + version int64 + height int8 + size int64 + hash []byte + leftHash []byte + leftNode *Node + rightHash []byte + rightNode *Node + persisted bool + persistedMem bool } // NewNode returns a new node from a key, value and version. diff --git a/nodedb.go b/nodedb.go index 2d97095c2..4948bc7db 100644 --- a/nodedb.go +++ b/nodedb.go @@ -33,9 +33,11 @@ var ( ) type nodeDB struct { - mtx sync.Mutex // Read/write lock. - db dbm.DB // Persistent node storage. - batch dbm.Batch // Batched writing buffer. + mtx sync.Mutex // Read/write lock. + db dbm.DB // Persistent node storage. + dbMem dbm.DB // Memory node storage. + batch dbm.Batch // Batched writing buffer. + memNodes map[string]*Node latestVersion int64 nodeCache map[string]*list.Element // Node cache. @@ -46,6 +48,8 @@ type nodeDB struct { func newNodeDB(db dbm.DB, cacheSize int) *nodeDB { ndb := &nodeDB{ db: db, + dbMem: dbm.NewMemDB(), + memNodes: map[string]*Node{}, batch: db.NewBatch(), latestVersion: 0, // initially invalid nodeCache: make(map[string]*list.Element), @@ -55,7 +59,7 @@ func newNodeDB(db dbm.DB, cacheSize int) *nodeDB { return ndb } -// GetNode gets a node from cache or disk. If it is an inner node, it does not +// GetNode gets a node from memory or disk. If it is an inner node, it does not // load its children. func (ndb *nodeDB) GetNode(hash []byte) *Node { ndb.mtx.Lock() @@ -72,32 +76,40 @@ func (ndb *nodeDB) GetNode(hash []byte) *Node { return elem.Value.(*Node) } - // Doesn't exist, load. - buf := ndb.db.Get(ndb.nodeKey(hash)) - if buf == nil { - panic(fmt.Sprintf("Value missing for hash %x corresponding to nodeKey %s", hash, ndb.nodeKey(hash))) - } + //Try reading from memory + var err error + node := ndb.memNodes[string(hash)] + if node == nil { + // Doesn't exist, load from disk + buf := ndb.db.Get(ndb.nodeKey(hash)) + if buf == nil { + panic(fmt.Sprintf("Value missing for hash %x corresponding to nodeKey %s", hash, ndb.nodeKey(hash))) + } - node, err := MakeNode(buf) - if err != nil { - panic(fmt.Sprintf("Error reading Node. bytes: %x, error: %v", buf, err)) + node, err = MakeNode(buf) + if err != nil { + panic(fmt.Sprintf("Error reading Node. bytes: %x, error: %v", buf, err)) + } + node.persisted = true } node.hash = hash - node.persisted = true ndb.cacheNode(node) return node } // SaveNode saves a node to disk. -func (ndb *nodeDB) SaveNode(node *Node) { +func (ndb *nodeDB) SaveNode(node *Node, flushToDisk bool) { ndb.mtx.Lock() defer ndb.mtx.Unlock() if node.hash == nil { panic("Expected to find node.hash, but none found.") } + if node.persistedMem && !flushToDisk { + return + } if node.persisted { panic("Shouldn't be calling save on an already persisted node.") } @@ -107,11 +119,14 @@ func (ndb *nodeDB) SaveNode(node *Node) { if err := node.writeBytes(buf); err != nil { panic(err) } - ndb.batch.Set(ndb.nodeKey(node.hash), buf.Bytes()) - debug("BATCH SAVE %X %p\n", node.hash, node) - node.persisted = true - ndb.cacheNode(node) + if flushToDisk { + ndb.batch.Set(ndb.nodeKey(node.hash), buf.Bytes()) + node.persisted = true + } else { + node.persistedMem = true + ndb.memNodes[string(node.hash)] = node + } } // Has checks if a hash exists in the database. @@ -132,23 +147,28 @@ func (ndb *nodeDB) Has(hash []byte) bool { // NOTE: This function clears leftNode/rigthNode recursively and // calls _hash() on the given node. // TODO refactor, maybe use hashWithCount() but provide a callback. -func (ndb *nodeDB) SaveBranch(node *Node) []byte { +func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { if node.persisted { return node.hash } + if node.persistedMem && !flushToDisk { + return node.hash + } if node.leftNode != nil { - node.leftHash = ndb.SaveBranch(node.leftNode) + node.leftHash = ndb.SaveBranch(node.leftNode, flushToDisk) } if node.rightNode != nil { - node.rightHash = ndb.SaveBranch(node.rightNode) + node.rightHash = ndb.SaveBranch(node.rightNode, flushToDisk) } node._hash() - ndb.SaveNode(node) + ndb.SaveNode(node, flushToDisk) - node.leftNode = nil - node.rightNode = nil + if flushToDisk { + node.leftNode = nil + node.rightNode = nil + } return node.hash } @@ -165,11 +185,16 @@ func (ndb *nodeDB) DeleteVersion(version int64, checkLatestVersion bool) { // Saves orphaned nodes to disk under a special prefix. // version: the new version being saved. // orphans: the orphan nodes created since version-1 -func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { +func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64, flushToDisk bool) { ndb.mtx.Lock() defer ndb.mtx.Unlock() - toVersion := ndb.getPreviousVersion(version) + var toVersion int64 + toVersion = ndb.getPreviousVersioni(version, ndb.dbMem) + if toVersion == 0 { + toVersion = ndb.getPreviousVersion(version) + } //see if we have something on disk if we dont have anything from mem + for hash, fromVersion := range orphans { debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) ndb.saveOrphan([]byte(hash), fromVersion, toVersion) @@ -249,7 +274,11 @@ func (ndb *nodeDB) resetLatestVersion(version int64) { } func (ndb *nodeDB) getPreviousVersion(version int64) int64 { - itr := ndb.db.ReverseIterator( + return ndb.getPreviousVersionFromDB(version, ndb.db) +} + +func (ndb *nodeDB) getPreviousVersionFromDB(version int64, db dbm.DB) int64 { + itr := db.ReverseIterator( rootKeyFormat.Key(1), rootKeyFormat.Key(version), ) @@ -330,11 +359,15 @@ func (ndb *nodeDB) Commit() { defer ndb.mtx.Unlock() ndb.batch.Write() - ndb.batch.Close() ndb.batch = ndb.db.NewBatch() } func (ndb *nodeDB) getRoot(version int64) []byte { + memroot := ndb.dbMem.Get(ndb.rootKey(version)) + if len(memroot) > 0 { + return memroot + } + return ndb.db.Get(ndb.rootKey(version)) } From 23f0b990b308f0e932fbd541c1dc77f6ec580fb9 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:27:14 +0700 Subject: [PATCH 02/57] Add version saving to in memory, and ability to flush to disk periodically --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38933bf95..5e6b78df4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Pending + +- [IAVL] Ability to store temporary versions in memory only, and flush to disk every X versions. This should greatly reduce IO requirements and disk storage. (@mattkanwisher Loom Network) + ## 0.12.2 (March 13, 2019) IMPROVEMENTS From aef79a7cb3a64039369a4f2a61ee8db264b27240 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:53:23 +0700 Subject: [PATCH 03/57] Add version saving to in memory, and ability to flush to disk periodically --- nodedb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodedb.go b/nodedb.go index 4948bc7db..c35ac8ab7 100644 --- a/nodedb.go +++ b/nodedb.go @@ -190,7 +190,7 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64, flushToD defer ndb.mtx.Unlock() var toVersion int64 - toVersion = ndb.getPreviousVersioni(version, ndb.dbMem) + toVersion = ndb.getPreviousVersionDB(version, ndb.dbMem) if toVersion == 0 { toVersion = ndb.getPreviousVersion(version) } //see if we have something on disk if we dont have anything from mem From 43d2409a5e3a3e7911a8e8d460e385764b8a55f1 Mon Sep 17 00:00:00 2001 From: Matthew Campbell Date: Tue, 2 Jul 2019 16:54:57 +0700 Subject: [PATCH 04/57] Add version saving to in memory, and ability to flush to disk periodically --- nodedb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodedb.go b/nodedb.go index c35ac8ab7..ca3a797eb 100644 --- a/nodedb.go +++ b/nodedb.go @@ -190,7 +190,7 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64, flushToD defer ndb.mtx.Unlock() var toVersion int64 - toVersion = ndb.getPreviousVersionDB(version, ndb.dbMem) + toVersion = ndb.getPreviousVersionFromDB(version, ndb.dbMem) if toVersion == 0 { toVersion = ndb.getPreviousVersion(version) } //see if we have something on disk if we dont have anything from mem From c6f7eee70789f2ed8f65014d0c9f40795262da7a Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 8 Jul 2019 15:56:38 -0700 Subject: [PATCH 05/57] initial changes --- mutable_tree.go | 48 ++++++++++++++++++++++++++--------------- node.go | 2 +- nodedb.go | 57 ++++++++++++++++++++++++++++--------------------- 3 files changed, 65 insertions(+), 42 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index 1f99b24ca..8384dcb2b 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -16,13 +16,15 @@ type MutableTree struct { *ImmutableTree // The current, working tree. lastSaved *ImmutableTree // The most recently saved tree. orphans map[string]int64 // Nodes removed by changes to working tree. - memVersions map[int64]bool // The previous, saved versions of the tree in mem. - versions map[int64]bool // The previous, saved versions of the tree. + versions map[int64]bool // The previous versions of the tree saved in disk or memory. ndb *nodeDB + // Pruning fields + keepEvery int64 // Saves version to disk periodically + keepRecent int64 // Saves recent versions in memory } -// NewMutableTree returns a new tree with the specified cache size and datastore. -func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { +// NewMutableTree returns a new tree with the specified cache size and datastore and pruning options +func NewMutableTree(db dbm.DB, cacheSize int, keepEvery, keepRecent int64) *MutableTree { ndb := newNodeDB(db, cacheSize) head := &ImmutableTree{ndb: ndb} @@ -30,9 +32,10 @@ func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { ImmutableTree: head, lastSaved: head.clone(), orphans: map[string]int64{}, - memVersions: map[int64]bool{}, versions: map[int64]bool{}, ndb: ndb, + keepEvery: keepEvery, + keepRecent: keepRecent, } } @@ -366,6 +369,10 @@ func (tree *MutableTree) GetVersioned(key []byte, version int64) ( return -1, nil } +func (tree *MutableTree) saveToDisk(version int64) bool { + return tree.keepEvery != 0 && version%tree.keepEvery == 0 +} + // SaveVersionMem saves a new tree version to disk, based on the current state of // the tree. Returns the hash and new version number. func (tree *MutableTree) SaveVersionMem() ([]byte, int64, error) { @@ -374,28 +381,26 @@ func (tree *MutableTree) SaveVersionMem() ([]byte, int64, error) { // FlushMemDisk saves a new tree to disk and removes all the versions in memory func (tree *MutableTree) FlushMemVersionDisk() ([]byte, int64, error) { - x, y, err := tree.saveVersion(true) + hash, version, err := tree.saveVersion(true) tree.ndb.dbMem = dbm.NewMemDB() - tree.memVersions = map[int64]bool{} + tree.ndb.memNodes = map[string]*Node{} - return x, y, err + return hash, version, err } -// SaveVersion saves a new tree version to disk, based on the current state of -// the tree. Returns the hash and new version number. +// SaveVersion saves a new tree version to memDB and removes old version, +// based on the current state of the tree. Returns the hash and new version number. +// If version is snapshot version, persist version to disk as well func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { - return tree.saveVersion(true) -} - -func (tree *MutableTree) saveVersion(flushToDisk bool) ([]byte, int64, error) { version := tree.version + 1 + // check if version needs to be flushed to disk as well + flushToDisk := saveToDisk(version) + if flushToDisk { tree.ndb.batch = tree.ndb.db.NewBatch() - } else { - tree.ndb.batch = tree.ndb.dbMem.NewBatch() - tree.memVersions[version] = true } + tree.ndb.memBatch = tree.ndb.dbMem.NewBatch() if tree.versions[version] { //version already exists, throw an error if attempting to overwrite @@ -412,6 +417,10 @@ func (tree *MutableTree) saveVersion(flushToDisk bool) ([]byte, int64, error) { return nil, version, fmt.Errorf("version %d was already saved to different hash %X (existing hash %X)", version, newHash, existingHash) } + tree.versions[version] = true + + // Delete oldest recent version in memDB + tree.DeleteVersionMem(version) if tree.root == nil { // There can still be orphans, for example if the root is the node being @@ -478,6 +487,11 @@ func (tree *MutableTree) DeleteVersionFull(version int64, memDeleteAlso bool) er return nil } +func DeleteVersionMem(version int64) error { + //tree.ndb.batchMem = tree.ndb.dbMem.NewBatch() + return nil +} + // deleteVersionsFrom deletes tree version from disk specified version to latest version. The version can then no // longer be accessed. func (tree *MutableTree) deleteVersionsFrom(version int64) error { diff --git a/node.go b/node.go index c039401ba..087f6efaf 100644 --- a/node.go +++ b/node.go @@ -25,8 +25,8 @@ type Node struct { leftNode *Node rightHash []byte rightNode *Node + saved bool persisted bool - persistedMem bool } // NewNode returns a new node from a key, value and version. diff --git a/nodedb.go b/nodedb.go index ca3a797eb..dfeed0041 100644 --- a/nodedb.go +++ b/nodedb.go @@ -35,9 +35,9 @@ var ( type nodeDB struct { mtx sync.Mutex // Read/write lock. db dbm.DB // Persistent node storage. - dbMem dbm.DB // Memory node storage. + memDb dbm.DB // Memory node storage. batch dbm.Batch // Batched writing buffer. - memNodes map[string]*Node + memBatch dbm.Batch // Batched writing buffer for memDB. latestVersion int64 nodeCache map[string]*list.Element // Node cache. @@ -48,9 +48,9 @@ type nodeDB struct { func newNodeDB(db dbm.DB, cacheSize int) *nodeDB { ndb := &nodeDB{ db: db, - dbMem: dbm.NewMemDB(), - memNodes: map[string]*Node{}, + memDb: dbm.NewMemDB(), batch: db.NewBatch(), + memBatch: memDb.NewBatch(), latestVersion: 0, // initially invalid nodeCache: make(map[string]*list.Element), nodeCacheSize: cacheSize, @@ -77,21 +77,23 @@ func (ndb *nodeDB) GetNode(hash []byte) *Node { } //Try reading from memory - var err error - node := ndb.memNodes[string(hash)] - if node == nil { + buf := ndb.memDb.Get(ndb.nodeKey(hash)) + persisted := false + if buf == nil { // Doesn't exist, load from disk buf := ndb.db.Get(ndb.nodeKey(hash)) if buf == nil { panic(fmt.Sprintf("Value missing for hash %x corresponding to nodeKey %s", hash, ndb.nodeKey(hash))) } + persisted = true + } - node, err = MakeNode(buf) - if err != nil { - panic(fmt.Sprintf("Error reading Node. bytes: %x, error: %v", buf, err)) - } - node.persisted = true + node, err = MakeNode(buf) + if err != nil { + panic(fmt.Sprintf("Error reading Node. bytes: %x, error: %v", buf, err)) } + node.saved = true + node.persisted = true node.hash = hash ndb.cacheNode(node) @@ -123,18 +125,23 @@ func (ndb *nodeDB) SaveNode(node *Node, flushToDisk bool) { if flushToDisk { ndb.batch.Set(ndb.nodeKey(node.hash), buf.Bytes()) node.persisted = true - } else { - node.persistedMem = true - ndb.memNodes[string(node.hash)] = node } + + node.saved = true + ndb.memBatch.Set(ndb.nodeKey(node.hash), buf.Bytes()) } // Has checks if a hash exists in the database. func (ndb *nodeDB) Has(hash []byte) bool { key := ndb.nodeKey(hash) + exists, err := ndb.memDb.Has(key, nil) + if exists { + return exists + } + if ldb, ok := ndb.db.(*dbm.GoLevelDB); ok { - exists, err := ldb.DB().Has(key, nil) + exists, err = ldb.DB().Has(key, nil) if err != nil { panic("Got error from leveldb: " + err.Error()) } @@ -148,10 +155,7 @@ func (ndb *nodeDB) Has(hash []byte) bool { // calls _hash() on the given node. // TODO refactor, maybe use hashWithCount() but provide a callback. func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { - if node.persisted { - return node.hash - } - if node.persistedMem && !flushToDisk { + if node.saved { return node.hash } @@ -197,17 +201,20 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64, flushToD for hash, fromVersion := range orphans { debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) - ndb.saveOrphan([]byte(hash), fromVersion, toVersion) + ndb.saveOrphan([]byte(hash), fromVersion, toVersion, flushToDisk) } } -// Saves a single orphan to disk. -func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64) { +// Saves a single orphan to memDB. If flushToDisk, persist to disk as well. +func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushToDisk bool) { if fromVersion > toVersion { panic(fmt.Sprintf("Orphan expires before it comes alive. %d > %d", fromVersion, toVersion)) } key := ndb.orphanKey(fromVersion, toVersion, hash) - ndb.batch.Set(key, hash) + if flushToDisk { + ndb.batch.Set(key, hash) + } + ndb.memBatch.Set(key, hash) } // deleteOrphans deletes orphaned nodes from disk, and the associated orphan @@ -359,7 +366,9 @@ func (ndb *nodeDB) Commit() { defer ndb.mtx.Unlock() ndb.batch.Write() + ndb.memBatch.Write() ndb.batch = ndb.db.NewBatch() + ndb.memBatch = ndb.dbMem.NewBatch() } func (ndb *nodeDB) getRoot(version int64) []byte { From 099ec059c6d7c5566c3e306b9c6bca7946820f10 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Wed, 10 Jul 2019 15:37:21 -0700 Subject: [PATCH 06/57] more changes to nodedb --- nodedb.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/nodedb.go b/nodedb.go index dfeed0041..475bd8776 100644 --- a/nodedb.go +++ b/nodedb.go @@ -38,6 +38,7 @@ type nodeDB struct { memDb dbm.DB // Memory node storage. batch dbm.Batch // Batched writing buffer. memBatch dbm.Batch // Batched writing buffer for memDB. + memVersions int64 // number of versions in memDB latestVersion int64 nodeCache map[string]*list.Element // Node cache. @@ -45,12 +46,13 @@ type nodeDB struct { nodeCacheQueue *list.List // LRU queue of cache elements. Used for deletion. } -func newNodeDB(db dbm.DB, cacheSize int) *nodeDB { +func newNodeDB(db dbm.DB, cacheSize int, versionsInMem int64) *nodeDB { ndb := &nodeDB{ db: db, memDb: dbm.NewMemDB(), batch: db.NewBatch(), memBatch: memDb.NewBatch(), + memVersions: versionsInMem, latestVersion: 0, // initially invalid nodeCache: make(map[string]*list.Element), nodeCacheSize: cacheSize, @@ -93,10 +95,13 @@ func (ndb *nodeDB) GetNode(hash []byte) *Node { panic(fmt.Sprintf("Error reading Node. bytes: %x, error: %v", buf, err)) } node.saved = true - node.persisted = true + node.persisted = persisted node.hash = hash - ndb.cacheNode(node) + // only cache if we pulled node from disk + if node.persisted { + ndb.cacheNode(node) + } return node } @@ -109,7 +114,7 @@ func (ndb *nodeDB) SaveNode(node *Node, flushToDisk bool) { if node.hash == nil { panic("Expected to find node.hash, but none found.") } - if node.persistedMem && !flushToDisk { + if node.saved && !flushToDisk { return } if node.persisted { @@ -189,15 +194,19 @@ func (ndb *nodeDB) DeleteVersion(version int64, checkLatestVersion bool) { // Saves orphaned nodes to disk under a special prefix. // version: the new version being saved. // orphans: the orphan nodes created since version-1 -func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64, flushToDisk bool) { +func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { ndb.mtx.Lock() defer ndb.mtx.Unlock() var toVersion int64 - toVersion = ndb.getPreviousVersionFromDB(version, ndb.dbMem) - if toVersion == 0 { - toVersion = ndb.getPreviousVersion(version) - } //see if we have something on disk if we dont have anything from mem + var flushToDisk bool + toVersion = ndb.getPreviousVersionFromDB(version, ndb.memDb) + if toVersion { + flushToDisk = false + } else { + toVersion = ndb.getPreviousVersionFromDB(version, ndb.db) + flushToDisk = true + } for hash, fromVersion := range orphans { debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) @@ -213,8 +222,9 @@ func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushTo key := ndb.orphanKey(fromVersion, toVersion, hash) if flushToDisk { ndb.batch.Set(key, hash) + } else { + ndb.memBatch.Set(key, hash) } - ndb.memBatch.Set(key, hash) } // deleteOrphans deletes orphaned nodes from disk, and the associated orphan @@ -246,7 +256,13 @@ func (ndb *nodeDB) deleteOrphans(version int64) { ndb.uncacheNode(hash) } else { debug("MOVE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) - ndb.saveOrphan(hash, fromVersion, predecessor) + var flushToDisk bool + if predecessor > latestVersion - memVersions { + flushToDisk = false + } else { + flushToDisk = true + } + ndb.saveOrphan(hash, fromVersion, predecessor, flushToDisk) } }) } @@ -281,6 +297,13 @@ func (ndb *nodeDB) resetLatestVersion(version int64) { } func (ndb *nodeDB) getPreviousVersion(version int64) int64 { + // If version exists in memDB, check memDB for any previous version + if (version > ndb.latestVersion - ndb.memVersions) { + prev := getPreviousVersionFromDB(version, ndb.memDb) + if prev { + return prev + } + } return ndb.getPreviousVersionFromDB(version, ndb.db) } @@ -317,11 +340,23 @@ func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { // Traverse orphans ending at a certain version. func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { - ndb.traversePrefix(orphanKeyFormat.Key(version), fn) + prefix := orphanKeyFormat.Key(version) + if version > ndb.latestVersion - memVersions { + ndb.traversePrefixFromDB(ndb.memDb, prefix, fn) + } else { + ndb.traversePrefixFromDB(ndb.db, prefix, fn) + } } -// Traverse all keys. +// Traverse all keys from memDB and disk DB func (ndb *nodeDB) traverse(fn func(key, value []byte)) { + memItr := ndb.memDb.Iterator(nil, nil) + defer memItr.Close() + + for ; memItr.Valid(); memItr.Next() { + fn(memItr.Key(), memItr.Value()) + } + itr := ndb.db.Iterator(nil, nil) defer itr.Close() @@ -330,8 +365,25 @@ func (ndb *nodeDB) traverse(fn func(key, value []byte)) { } } -// Traverse all keys with a certain prefix. +// Traverse all keys from provided DB +func traverseFromDB(db dbm.DB, fn (func key, value []byte)) { + itr := db.Iterator(nil, nil) + defer itr.Close() + + for ; itr.Valid(); itr.Next() { + fn(itr.Key(), itr.Value()) + } +} + +// Traverse all keys with a certain prefix from memDB and disk DB func (ndb *nodeDB) traversePrefix(prefix []byte, fn func(k, v []byte)) { + memItr := dbm.IteratePrefix(ndb.memDb, prefix) + defer memItr.Close() + + for ; memItr.Valid(); memItr.Next() { + fn(memItr.Key(), memItr.Value()) + } + itr := dbm.IteratePrefix(ndb.db, prefix) defer itr.Close() @@ -340,6 +392,16 @@ func (ndb *nodeDB) traversePrefix(prefix []byte, fn func(k, v []byte)) { } } +// Traverse all keys with a certain prefix from given DB +func traversePrefixFromDB(db dbm.DB, prefix []byte, fn func(k, v []byte)) { + itr := dbm.IteratePrefix(db, prefix) + defer itr.Close() + + for ; itr.Valid(); itr.Next() { + fn(itr.Key(), itr.Value()) + } +} + func (ndb *nodeDB) uncacheNode(hash []byte) { if elem, ok := ndb.nodeCache[string(hash)]; ok { ndb.nodeCacheQueue.Remove(elem) From 6dc8e62c62c1e68f6fb206724f90fd03f0b2568b Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Wed, 10 Jul 2019 15:43:46 -0700 Subject: [PATCH 07/57] fmt and some docs --- mutable_tree.go | 6 +++--- node.go | 24 ++++++++++++------------ nodedb.go | 20 ++++++++++---------- nodedb_test.go | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index 8384dcb2b..9dd63a230 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -34,8 +34,8 @@ func NewMutableTree(db dbm.DB, cacheSize int, keepEvery, keepRecent int64) *Muta orphans: map[string]int64{}, versions: map[int64]bool{}, ndb: ndb, - keepEvery: keepEvery, - keepRecent: keepRecent, + keepEvery: keepEvery, + keepRecent: keepRecent, } } @@ -388,7 +388,7 @@ func (tree *MutableTree) FlushMemVersionDisk() ([]byte, int64, error) { return hash, version, err } -// SaveVersion saves a new tree version to memDB and removes old version, +// SaveVersion saves a new tree version to memDB and removes old version, // based on the current state of the tree. Returns the hash and new version number. // If version is snapshot version, persist version to disk as well func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { diff --git a/node.go b/node.go index 087f6efaf..4e0a320af 100644 --- a/node.go +++ b/node.go @@ -15,18 +15,18 @@ import ( // Node represents a node in a Tree. type Node struct { - key []byte - value []byte - version int64 - height int8 - size int64 - hash []byte - leftHash []byte - leftNode *Node - rightHash []byte - rightNode *Node - saved bool - persisted bool + key []byte + value []byte + version int64 + height int8 + size int64 + hash []byte + leftHash []byte + leftNode *Node + rightHash []byte + rightNode *Node + saved bool // saved to memory or disk + persisted bool // persisted to disk } // NewNode returns a new node from a key, value and version. diff --git a/nodedb.go b/nodedb.go index 475bd8776..fe30f16d7 100644 --- a/nodedb.go +++ b/nodedb.go @@ -33,12 +33,12 @@ var ( ) type nodeDB struct { - mtx sync.Mutex // Read/write lock. - db dbm.DB // Persistent node storage. - memDb dbm.DB // Memory node storage. - batch dbm.Batch // Batched writing buffer. - memBatch dbm.Batch // Batched writing buffer for memDB. - memVersions int64 // number of versions in memDB + mtx sync.Mutex // Read/write lock. + db dbm.DB // Persistent node storage. + memDb dbm.DB // Memory node storage. + batch dbm.Batch // Batched writing buffer. + memBatch dbm.Batch // Batched writing buffer for memDB. + memVersions int64 // number of versions in memDB latestVersion int64 nodeCache map[string]*list.Element // Node cache. @@ -257,7 +257,7 @@ func (ndb *nodeDB) deleteOrphans(version int64) { } else { debug("MOVE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) var flushToDisk bool - if predecessor > latestVersion - memVersions { + if predecessor > latestVersion-memVersions { flushToDisk = false } else { flushToDisk = true @@ -298,7 +298,7 @@ func (ndb *nodeDB) resetLatestVersion(version int64) { func (ndb *nodeDB) getPreviousVersion(version int64) int64 { // If version exists in memDB, check memDB for any previous version - if (version > ndb.latestVersion - ndb.memVersions) { + if version > ndb.latestVersion-ndb.memVersions { prev := getPreviousVersionFromDB(version, ndb.memDb) if prev { return prev @@ -341,7 +341,7 @@ func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { // Traverse orphans ending at a certain version. func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { prefix := orphanKeyFormat.Key(version) - if version > ndb.latestVersion - memVersions { + if version > ndb.latestVersion-memVersions { ndb.traversePrefixFromDB(ndb.memDb, prefix, fn) } else { ndb.traversePrefixFromDB(ndb.db, prefix, fn) @@ -366,7 +366,7 @@ func (ndb *nodeDB) traverse(fn func(key, value []byte)) { } // Traverse all keys from provided DB -func traverseFromDB(db dbm.DB, fn (func key, value []byte)) { +func traverseFromDB(db dbm.DB, fn func(key, value []byte)) { itr := db.Iterator(nil, nil) defer itr.Close() diff --git a/nodedb_test.go b/nodedb_test.go index eee0ce16b..8e891a15e 100644 --- a/nodedb_test.go +++ b/nodedb_test.go @@ -26,7 +26,7 @@ func makeHashes(b *testing.B, seed int64) [][]byte { b.StopTimer() rnd := rand.NewSource(seed) hashes := make([][]byte, b.N) - hashBytes := 8*((hashSize+7)/8) + hashBytes := 8 * ((hashSize + 7) / 8) for i := 0; i < b.N; i++ { hashes[i] = make([]byte, hashBytes) for b := 0; b < hashBytes; b += 8 { From 6f51bea20e8e19dbed884d28d5ac824aa53f7575 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Wed, 10 Jul 2019 17:18:25 -0700 Subject: [PATCH 08/57] move some pruning logic into nodedb --- mutable_tree.go | 5 +--- nodedb.go | 61 ++++++++++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index 9dd63a230..fd88928bb 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -18,14 +18,11 @@ type MutableTree struct { orphans map[string]int64 // Nodes removed by changes to working tree. versions map[int64]bool // The previous versions of the tree saved in disk or memory. ndb *nodeDB - // Pruning fields - keepEvery int64 // Saves version to disk periodically - keepRecent int64 // Saves recent versions in memory } // NewMutableTree returns a new tree with the specified cache size and datastore and pruning options func NewMutableTree(db dbm.DB, cacheSize int, keepEvery, keepRecent int64) *MutableTree { - ndb := newNodeDB(db, cacheSize) + ndb := newNodeDB(db, cacheSize, keepRecent, keepEvery) head := &ImmutableTree{ndb: ndb} return &MutableTree{ diff --git a/nodedb.go b/nodedb.go index fe30f16d7..0635e4d1c 100644 --- a/nodedb.go +++ b/nodedb.go @@ -33,12 +33,13 @@ var ( ) type nodeDB struct { - mtx sync.Mutex // Read/write lock. - db dbm.DB // Persistent node storage. - memDb dbm.DB // Memory node storage. - batch dbm.Batch // Batched writing buffer. - memBatch dbm.Batch // Batched writing buffer for memDB. - memVersions int64 // number of versions in memDB + mtx sync.Mutex // Read/write lock. + db dbm.DB // Persistent node storage. + memDb dbm.DB // Memory node storage. + batch dbm.Batch // Batched writing buffer. + memBatch dbm.Batch // Batched writing buffer for memDB. + keepRecent int64 // number of recent versions to store in memDB + keepEvery int64 // frequency of snaphots saved to disk latestVersion int64 nodeCache map[string]*list.Element // Node cache. @@ -46,13 +47,14 @@ type nodeDB struct { nodeCacheQueue *list.List // LRU queue of cache elements. Used for deletion. } -func newNodeDB(db dbm.DB, cacheSize int, versionsInMem int64) *nodeDB { +func newNodeDB(db dbm.DB, cacheSize int, keepRecent, keepEvery int64) *nodeDB { ndb := &nodeDB{ db: db, memDb: dbm.NewMemDB(), batch: db.NewBatch(), memBatch: memDb.NewBatch(), - memVersions: versionsInMem, + keepRecent: keepRecent, + keepEvery: keepEvery, latestVersion: 0, // initially invalid nodeCache: make(map[string]*list.Element), nodeCacheSize: cacheSize, @@ -174,10 +176,8 @@ func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { node._hash() ndb.SaveNode(node, flushToDisk) - if flushToDisk { - node.leftNode = nil - node.rightNode = nil - } + node.leftNode = nil + node.rightNode = nil return node.hash } @@ -191,6 +191,14 @@ func (ndb *nodeDB) DeleteVersion(version int64, checkLatestVersion bool) { ndb.deleteRoot(version, checkLatestVersion) } +func (ndb *nodeDB) containsSnapshot(fromVersion, toVersion int64) bool { + return toVersion/ndb.keepEvery != fromVersion/ndb.keepEvery +} + +func (ndb *nodeDB) versionInMemDB(version int64) { + return version > ndb.latestVersion-ndb.keepRecent +} + // Saves orphaned nodes to disk under a special prefix. // version: the new version being saved. // orphans: the orphan nodes created since version-1 @@ -199,17 +207,12 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { defer ndb.mtx.Unlock() var toVersion int64 - var flushToDisk bool toVersion = ndb.getPreviousVersionFromDB(version, ndb.memDb) - if toVersion { - flushToDisk = false - } else { - toVersion = ndb.getPreviousVersionFromDB(version, ndb.db) - flushToDisk = true - } for hash, fromVersion := range orphans { debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) + // if snapshot version in between fromVersion and toVersion, then flush to disk + flushToDisk := containsSnapshot(fromVersion, toVersion) ndb.saveOrphan([]byte(hash), fromVersion, toVersion, flushToDisk) } } @@ -222,13 +225,13 @@ func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushTo key := ndb.orphanKey(fromVersion, toVersion, hash) if flushToDisk { ndb.batch.Set(key, hash) - } else { - ndb.memBatch.Set(key, hash) } + ndb.memBatch.Set(key, hash) } // deleteOrphans deletes orphaned nodes from disk, and the associated orphan // entries. +// TODO: fix this function func (ndb *nodeDB) deleteOrphans(version int64) { // Will be zero if there is no previous version. predecessor := ndb.getPreviousVersion(version) @@ -243,7 +246,12 @@ func (ndb *nodeDB) deleteOrphans(version int64) { orphanKeyFormat.Scan(key, &toVersion, &fromVersion) // Delete orphan key and reverse-lookup key. - ndb.batch.Delete(key) + if containsSnapshot(fromVersion, toVersion) { + ndb.batch.Delete(key) + } + if versionInMemDB(toVersion) { + ndb.memBatch.Delete(key) + } // If there is no predecessor, or the predecessor is earlier than the // beginning of the lifetime (ie: negative lifetime), or the lifetime @@ -257,7 +265,7 @@ func (ndb *nodeDB) deleteOrphans(version int64) { } else { debug("MOVE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) var flushToDisk bool - if predecessor > latestVersion-memVersions { + if predecessor > latestVersion-ndb.keepRecent { flushToDisk = false } else { flushToDisk = true @@ -298,7 +306,7 @@ func (ndb *nodeDB) resetLatestVersion(version int64) { func (ndb *nodeDB) getPreviousVersion(version int64) int64 { // If version exists in memDB, check memDB for any previous version - if version > ndb.latestVersion-ndb.memVersions { + if version > ndb.latestVersion-ndb.ndb.keepRecent { prev := getPreviousVersionFromDB(version, ndb.memDb) if prev { return prev @@ -341,9 +349,10 @@ func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { // Traverse orphans ending at a certain version. func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { prefix := orphanKeyFormat.Key(version) - if version > ndb.latestVersion-memVersions { + if version > ndb.latestVersion-ndb.keepRecent { ndb.traversePrefixFromDB(ndb.memDb, prefix, fn) - } else { + } + if version%keepEvery == 0 { ndb.traversePrefixFromDB(ndb.db, prefix, fn) } } From 0e2a95421cd10a1f3d8a3a2a7196f308667eb359 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Thu, 11 Jul 2019 13:07:27 -0700 Subject: [PATCH 09/57] completed changes to nodedb --- nodedb.go | 243 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 138 insertions(+), 105 deletions(-) diff --git a/nodedb.go b/nodedb.go index 0635e4d1c..49e97b271 100644 --- a/nodedb.go +++ b/nodedb.go @@ -33,13 +33,13 @@ var ( ) type nodeDB struct { - mtx sync.Mutex // Read/write lock. - db dbm.DB // Persistent node storage. - memDb dbm.DB // Memory node storage. - batch dbm.Batch // Batched writing buffer. - memBatch dbm.Batch // Batched writing buffer for memDB. - keepRecent int64 // number of recent versions to store in memDB - keepEvery int64 // frequency of snaphots saved to disk + mtx sync.Mutex // Read/write lock. + snapshotDB dbm.DB // Persistent node storage. + recentDB dbm.DB // Memory node storage. + snapshotBatch dbm.Batch // Batched writing buffer. + recentBatch dbm.Batch // Batched writing buffer for recentDB. + keepRecent int64 // number of recent versions to store in recentDB + keepEvery int64 // frequency of snaphots saved to disk latestVersion int64 nodeCache map[string]*list.Element // Node cache. @@ -47,12 +47,12 @@ type nodeDB struct { nodeCacheQueue *list.List // LRU queue of cache elements. Used for deletion. } -func newNodeDB(db dbm.DB, cacheSize int, keepRecent, keepEvery int64) *nodeDB { +func newNodeDB(snapshotDB dbm.DB, recentDB dmb.DB, cacheSize int, keepRecent, keepEvery int64) *nodeDB { ndb := &nodeDB{ - db: db, - memDb: dbm.NewMemDB(), - batch: db.NewBatch(), - memBatch: memDb.NewBatch(), + snapshotDB: snapshotDB, + recentDB: recentDB, + snapshotBatch: snapshotDB.NewBatch(), + recentBatch: recentDB.NewBatch(), keepRecent: keepRecent, keepEvery: keepEvery, latestVersion: 0, // initially invalid @@ -80,12 +80,12 @@ func (ndb *nodeDB) GetNode(hash []byte) *Node { return elem.Value.(*Node) } - //Try reading from memory - buf := ndb.memDb.Get(ndb.nodeKey(hash)) + //Try reading from recent memDB + buf := ndb.recentDB.Get(ndb.nodeKey(hash)) persisted := false if buf == nil { // Doesn't exist, load from disk - buf := ndb.db.Get(ndb.nodeKey(hash)) + buf := ndb.snapshotDB.Get(ndb.nodeKey(hash)) if buf == nil { panic(fmt.Sprintf("Value missing for hash %x corresponding to nodeKey %s", hash, ndb.nodeKey(hash))) } @@ -100,10 +100,7 @@ func (ndb *nodeDB) GetNode(hash []byte) *Node { node.persisted = persisted node.hash = hash - // only cache if we pulled node from disk - if node.persisted { - ndb.cacheNode(node) - } + ndb.cacheNode(node) return node } @@ -130,31 +127,31 @@ func (ndb *nodeDB) SaveNode(node *Node, flushToDisk bool) { } if flushToDisk { - ndb.batch.Set(ndb.nodeKey(node.hash), buf.Bytes()) + ndb.snapshotBatch.Set(ndb.nodeKey(node.hash), buf.Bytes()) node.persisted = true } node.saved = true - ndb.memBatch.Set(ndb.nodeKey(node.hash), buf.Bytes()) + ndb.recentBatch.Set(ndb.nodeKey(node.hash), buf.Bytes()) } // Has checks if a hash exists in the database. func (ndb *nodeDB) Has(hash []byte) bool { key := ndb.nodeKey(hash) - exists, err := ndb.memDb.Has(key, nil) - if exists { - return exists + val := ndb.recentDB.Get(key) + if val != nil { + return true } - if ldb, ok := ndb.db.(*dbm.GoLevelDB); ok { + if ldb, ok := ndb.snapshotDB.(*dbm.GoLevelDB); ok { exists, err = ldb.DB().Has(key, nil) if err != nil { panic("Got error from leveldb: " + err.Error()) } return exists } - return ndb.db.Get(key) != nil + return ndb.snapshotDB.Get(key) != nil } // SaveBranch saves the given node and all of its descendants. @@ -184,19 +181,19 @@ func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { // DeleteVersion deletes a tree version from disk. func (ndb *nodeDB) DeleteVersion(version int64, checkLatestVersion bool) { - ndb.mtx.Lock() - defer ndb.mtx.Unlock() - - ndb.deleteOrphans(version) - ndb.deleteRoot(version, checkLatestVersion) + deleteVersion(version, checkLatestVersion, false) } -func (ndb *nodeDB) containsSnapshot(fromVersion, toVersion int64) bool { - return toVersion/ndb.keepEvery != fromVersion/ndb.keepEvery +func (ndb *nodeDB) DeleteVersionFromRecent(version int64, checkLatestVersion bool) { + deleteVersion(version, checkLatestVersion, true) } -func (ndb *nodeDB) versionInMemDB(version int64) { - return version > ndb.latestVersion-ndb.keepRecent +func (ndb *nodeDB) deleteVersion(version int64, checkLatestVersion, memOnly bool) { + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + + ndb.deleteOrphans(version, memOnly) + ndb.deleteRoot(version, checkLatestVersion, memOnly) } // Saves orphaned nodes to disk under a special prefix. @@ -206,8 +203,7 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { ndb.mtx.Lock() defer ndb.mtx.Unlock() - var toVersion int64 - toVersion = ndb.getPreviousVersionFromDB(version, ndb.memDb) + toVersion := ndb.getPreviousVersionFromDB(version, ndb.recentDB) for hash, fromVersion := range orphans { debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) @@ -217,64 +213,74 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { } } -// Saves a single orphan to memDB. If flushToDisk, persist to disk as well. +// Saves a single orphan to recentDB. If flushToDisk, persist to disk as well. func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushToDisk bool) { if fromVersion > toVersion { panic(fmt.Sprintf("Orphan expires before it comes alive. %d > %d", fromVersion, toVersion)) } key := ndb.orphanKey(fromVersion, toVersion, hash) + ndb.recentBatch.Set(key, hash) if flushToDisk { - ndb.batch.Set(key, hash) + // save to disk with toVersion equal to snapshotVersion closest to original toVersion + snapVersion := toVersion - (toVersion%keepEvery) + key = ndb.orphanKey(fromVersion, snapVersion, hash) + ndb.snapshotBatch.Set(key, hash) } - ndb.memBatch.Set(key, hash) } // deleteOrphans deletes orphaned nodes from disk, and the associated orphan // entries. -// TODO: fix this function -func (ndb *nodeDB) deleteOrphans(version int64) { - // Will be zero if there is no previous version. - predecessor := ndb.getPreviousVersion(version) - - // Traverse orphans with a lifetime ending at the version specified. - // TODO optimize. - ndb.traverseOrphansVersion(version, func(key, hash []byte) { - var fromVersion, toVersion int64 - - // See comment on `orphanKeyFmt`. Note that here, `version` and - // `toVersion` are always equal. - orphanKeyFormat.Scan(key, &toVersion, &fromVersion) - - // Delete orphan key and reverse-lookup key. - if containsSnapshot(fromVersion, toVersion) { - ndb.batch.Delete(key) - } - if versionInMemDB(toVersion) { - ndb.memBatch.Delete(key) - } - - // If there is no predecessor, or the predecessor is earlier than the - // beginning of the lifetime (ie: negative lifetime), or the lifetime - // spans a single version and that version is the one being deleted, we - // can delete the orphan. Otherwise, we shorten its lifetime, by - // moving its endpoint to the previous version. - if predecessor < fromVersion || fromVersion == toVersion { - debug("DELETE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) - ndb.batch.Delete(ndb.nodeKey(hash)) +func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { + deleteOrphansMem(version) + if isSnapshotVersion(version) && !memOnly { + traverseOrphansVersionFromDB(ndb.snapshotDB, version, func(key, hash []byte) { + ndb.snapshotDB.Delete(key) + deleteOrphansHelper(ndb.snapshotDB, ndb.snapshotBatch, true, key, hash) + }) + } +} + +func (ndb *nodeDB) deleteOrphansMem(version int64) { + traverseOrphansVersionFromDB(ndb.recentDB, version, func(key, hash []byte) { + ndb.recentBatch.Delete(key) + // common case, we are deleting orphans from least recent version that is getting pruned from memDB + if version == ndb.latestVersion-ndb.keepRecent { + // delete orphan look-up, delete and uncache node + ndb.recentBatch.Delete(ndb.nodeKey(hash)) ndb.uncacheNode(hash) - } else { - debug("MOVE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) - var flushToDisk bool - if predecessor > latestVersion-ndb.keepRecent { - flushToDisk = false - } else { - flushToDisk = true - } - ndb.saveOrphan(hash, fromVersion, predecessor, flushToDisk) + return } + + // user is manually deleting version from memDB + // thus predecessor may exist in memDB + // Will be zero if there is no previous version. + ndb.deleteOrphansHelper(ndb.recentDB, ndb.recentBatch, false, key, hash) }) } +func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk bool, key, hash []byte) { + var fromVersion, toVersion int64 + + // See comment on `orphanKeyFmt`. Note that here, `version` and + // `toVersion` are always equal. + orphanKeyFormat.Scan(key, &toVersion, &fromVersion) + + predecessor := getPreviousVersionFromDB(toVersion, ndb.recentDB) + // If there is no predecessor, or the predecessor is earlier than the + // beginning of the lifetime (ie: negative lifetime), or the lifetime + // spans a single version and that version is the one being deleted, we + // can delete the orphan. Otherwise, we shorten its lifetime, by + // moving its endpoint to the previous version. + if predecessor < fromVersion || fromVersion == toVersion { + debug("DELETE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) + batch.Delete(ndb.nodeKey(hash)) + ndb.uncacheNode(hash) + } else { + debug("MOVE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) + ndb.saveOrphan(hash, fromVersion, predecessor, flushToDisk) + } +} + func (ndb *nodeDB) nodeKey(hash []byte) []byte { return nodeKeyFormat.KeyBytes(hash) } @@ -305,14 +311,14 @@ func (ndb *nodeDB) resetLatestVersion(version int64) { } func (ndb *nodeDB) getPreviousVersion(version int64) int64 { - // If version exists in memDB, check memDB for any previous version - if version > ndb.latestVersion-ndb.ndb.keepRecent { - prev := getPreviousVersionFromDB(version, ndb.memDb) + // If version exists in recentDB, check recentDB for any previous version + if version > ndb.latestVersion-ndb.keepRecent { + prev := getPreviousVersionFromDB(version, ndb.recentDB) if prev { return prev } } - return ndb.getPreviousVersionFromDB(version, ndb.db) + return ndb.getPreviousVersionFromDB(version, ndb.snapshotDB) } func (ndb *nodeDB) getPreviousVersionFromDB(version int64, db dbm.DB) int64 { @@ -333,13 +339,16 @@ func (ndb *nodeDB) getPreviousVersionFromDB(version int64, db dbm.DB) int64 { } // deleteRoot deletes the root entry from disk, but not the node it points to. -func (ndb *nodeDB) deleteRoot(version int64, checkLatestVersion bool) { +func (ndb *nodeDB) deleteRoot(version int64, checkLatestVersion, memOnly bool) { if checkLatestVersion && version == ndb.getLatestVersion() { panic("Tried to delete latest version") } key := ndb.rootKey(version) - ndb.batch.Delete(key) + ndb.recentBatch.Delete(key) + if isSnapshotVersion(version) && !memOnly { + ndb.snapshotBatch.Delete(key) + } } func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { @@ -347,26 +356,33 @@ func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { } // Traverse orphans ending at a certain version. +// NOTE: If orphan is in recentDB and levelDB (version > latestVersion-keepRecent && version%keepEvery == 0) +// traverse will return the node twice. func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { prefix := orphanKeyFormat.Key(version) if version > ndb.latestVersion-ndb.keepRecent { - ndb.traversePrefixFromDB(ndb.memDb, prefix, fn) + traversePrefixFromDB(ndb.recentDB, prefix, fn) } if version%keepEvery == 0 { - ndb.traversePrefixFromDB(ndb.db, prefix, fn) + traversePrefixFromDB(ndb.snapshotDB, prefix, fn) } } -// Traverse all keys from memDB and disk DB +func traverseOrphansVersionFromDB(db dbm.DB, version int64, fn func(k, v []byte)) { + prefix := orphanKeyFormat.Key(version) + traversePrefixFromDB(db, prefix, fn) +} + +// Traverse all keys from recentDB and disk DB func (ndb *nodeDB) traverse(fn func(key, value []byte)) { - memItr := ndb.memDb.Iterator(nil, nil) + memItr := ndb.recentDB.Iterator(nil, nil) defer memItr.Close() for ; memItr.Valid(); memItr.Next() { fn(memItr.Key(), memItr.Value()) } - itr := ndb.db.Iterator(nil, nil) + itr := ndb.snapshotDB.Iterator(nil, nil) defer itr.Close() for ; itr.Valid(); itr.Next() { @@ -384,9 +400,9 @@ func traverseFromDB(db dbm.DB, fn func(key, value []byte)) { } } -// Traverse all keys with a certain prefix from memDB and disk DB +// Traverse all keys with a certain prefix from recentDB and disk DB func (ndb *nodeDB) traversePrefix(prefix []byte, fn func(k, v []byte)) { - memItr := dbm.IteratePrefix(ndb.memDb, prefix) + memItr := dbm.IteratePrefix(ndb.recentDB, prefix) defer memItr.Close() for ; memItr.Valid(); memItr.Next() { @@ -431,26 +447,32 @@ func (ndb *nodeDB) cacheNode(node *Node) { } } -// Write to disk. +// Write to disk and memDB func (ndb *nodeDB) Commit() { ndb.mtx.Lock() defer ndb.mtx.Unlock() - ndb.batch.Write() - ndb.memBatch.Write() - ndb.batch = ndb.db.NewBatch() - ndb.memBatch = ndb.dbMem.NewBatch() + ndb.snapshotBatch.Write() + ndb.snapshotBatch.Close() + ndb.recentBatch.Write() + ndb.recentBatch.Close() + ndb.snapshotBatch = ndb.snapshotDB.NewBatch() + ndb.recentBatch = ndb.dbMem.NewBatch() } func (ndb *nodeDB) getRoot(version int64) []byte { - memroot := ndb.dbMem.Get(ndb.rootKey(version)) - if len(memroot) > 0 { - return memroot + if isRecentVersion(version) { + memroot := ndb.recentDB.Get(ndb.rootKey(version)) + // TODO: maybe I shouldn't check in snapshot if it isn't here + if len(memroot) > 0 { + return memroot + } } - return ndb.db.Get(ndb.rootKey(version)) + return ndb.snapshotDB.Get(ndb.rootKey(version)) } +// NOTE: will get duplicated root if snapshot version exists in DB and memDB func (ndb *nodeDB) getRoots() (map[int64][]byte, error) { roots := map[int64][]byte{} @@ -468,15 +490,15 @@ func (ndb *nodeDB) SaveRoot(root *Node, version int64) error { if len(root.hash) == 0 { panic("Hash should not be empty") } - return ndb.saveRoot(root.hash, version) + return ndb.saveRoot(root.hash, version, isSnapshotVersion(version)) } // SaveEmptyRoot creates an entry on disk for an empty root. func (ndb *nodeDB) SaveEmptyRoot(version int64) error { - return ndb.saveRoot([]byte{}, version) + return ndb.saveRoot([]byte{}, version, isSnapshotVersion(version)) } -func (ndb *nodeDB) saveRoot(hash []byte, version int64) error { +func (ndb *nodeDB) saveRoot(hash []byte, version int64, flushToDisk bool) error { ndb.mtx.Lock() defer ndb.mtx.Unlock() @@ -485,14 +507,25 @@ func (ndb *nodeDB) saveRoot(hash []byte, version int64) error { } key := ndb.rootKey(version) - ndb.batch.Set(key, hash) ndb.updateLatestVersion(version) + ndb.recentBatch.Set(key, hash) + if flushToDisk { + ndb.snapshotBatch.Set(key, hash) + } return nil } ////////////////// Utility and test functions ///////////////////////////////// +func (ndb *nodeDB) isSnapshotVersion(version int64) bool { + return toVersion%ndb.keepEvery == 0 +} + +func (ndb *nodeDB) isRecentVersion(version int64) bool { + return version > ndb.latestVersion-ndb.keepRecent +} + func (ndb *nodeDB) leafNodes() []*Node { leaves := []*Node{} From a840877828fb32de25821b56947b0504f0e0e8d6 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Thu, 11 Jul 2019 18:02:44 -0700 Subject: [PATCH 10/57] initial updates to mutable_tree --- mutable_tree.go | 65 ++++++------------------------------------------- nodedb.go | 7 ++++++ 2 files changed, 15 insertions(+), 57 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index fd88928bb..4043ee596 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -31,8 +31,6 @@ func NewMutableTree(db dbm.DB, cacheSize int, keepEvery, keepRecent int64) *Muta orphans: map[string]int64{}, versions: map[int64]bool{}, ndb: ndb, - keepEvery: keepEvery, - keepRecent: keepRecent, } } @@ -366,39 +364,12 @@ func (tree *MutableTree) GetVersioned(key []byte, version int64) ( return -1, nil } -func (tree *MutableTree) saveToDisk(version int64) bool { - return tree.keepEvery != 0 && version%tree.keepEvery == 0 -} - -// SaveVersionMem saves a new tree version to disk, based on the current state of -// the tree. Returns the hash and new version number. -func (tree *MutableTree) SaveVersionMem() ([]byte, int64, error) { - return tree.saveVersion(false) -} - -// FlushMemDisk saves a new tree to disk and removes all the versions in memory -func (tree *MutableTree) FlushMemVersionDisk() ([]byte, int64, error) { - hash, version, err := tree.saveVersion(true) - tree.ndb.dbMem = dbm.NewMemDB() - - tree.ndb.memNodes = map[string]*Node{} - return hash, version, err -} - // SaveVersion saves a new tree version to memDB and removes old version, // based on the current state of the tree. Returns the hash and new version number. // If version is snapshot version, persist version to disk as well func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { version := tree.version + 1 - // check if version needs to be flushed to disk as well - flushToDisk := saveToDisk(version) - - if flushToDisk { - tree.ndb.batch = tree.ndb.db.NewBatch() - } - tree.ndb.memBatch = tree.ndb.dbMem.NewBatch() - if tree.versions[version] { //version already exists, throw an error if attempting to overwrite // Same hash means idempotent. Return success. @@ -416,22 +387,21 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { } tree.versions[version] = true - // Delete oldest recent version in memDB - tree.DeleteVersionMem(version) - if tree.root == nil { // There can still be orphans, for example if the root is the node being // removed. debug("SAVE EMPTY TREE %v\n", version) // Assume orphans not needed any more. So don't save any - tree.ndb.SaveOrphans(version, tree.orphans, flushToDisk) + tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveEmptyRoot(version) } else { debug("SAVE TREE %v\n", version) // Save the current tree. + // For now: flushToDisk is calculated in here. Ideally mutableTree has no notion of pruning + flushToDisk := version%tree.ndb.keepEvery == 0 tree.ndb.SaveBranch(tree.root, flushToDisk) // Assume orphans not needed any more. So don't save any - tree.ndb.SaveOrphans(version, tree.orphans, flushToDisk) + tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveRoot(tree.root, version) } tree.ndb.Commit() @@ -443,29 +413,15 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { tree.lastSaved = tree.ImmutableTree.clone() tree.orphans = map[string]int64{} + // Prune nodeDB + tree.ndb.PruneRecentVersions() + return tree.Hash(), version, nil } // DeleteVersion deletes a tree version from disk. The version can then no // longer be accessed. func (tree *MutableTree) DeleteVersion(version int64) error { - return tree.DeleteVersionFull(version, true) -} - -// DeleteVersionFull deletes a tree version from disk or memory based on the flag. The version can then no -// longer be accessed. -func (tree *MutableTree) DeleteVersionFull(version int64, memDeleteAlso bool) error { - if tree.memVersions[version] { - //sometimes you dont want to bother deleting versions in memory - if !memDeleteAlso { - return nil - } - tree.ndb.batch = tree.ndb.dbMem.NewBatch() - delete(tree.memVersions, version) - } else { - tree.ndb.batch = tree.ndb.db.NewBatch() - } - if version == 0 { return cmn.NewError("version must be greater than 0") } @@ -476,7 +432,7 @@ func (tree *MutableTree) DeleteVersionFull(version int64, memDeleteAlso bool) er return cmn.ErrorWrap(ErrVersionDoesNotExist, "") } - tree.ndb.DeleteVersion(version, true) + tree.ndb.DeleteVersion(version, tree.latestVersion) tree.ndb.Commit() delete(tree.versions, version) @@ -484,11 +440,6 @@ func (tree *MutableTree) DeleteVersionFull(version int64, memDeleteAlso bool) er return nil } -func DeleteVersionMem(version int64) error { - //tree.ndb.batchMem = tree.ndb.dbMem.NewBatch() - return nil -} - // deleteVersionsFrom deletes tree version from disk specified version to latest version. The version can then no // longer be accessed. func (tree *MutableTree) deleteVersionsFrom(version int64) error { diff --git a/nodedb.go b/nodedb.go index 49e97b271..a3056590a 100644 --- a/nodedb.go +++ b/nodedb.go @@ -281,6 +281,13 @@ func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk b } } +func (ndb *nodeDB) PruneRecentVersions() { + if ndb.latestVersion - ndb.keepRecent <= 0 { + return + } + ndb.DeleteVersionFromRecent(ndb.latestVersion-keepRecent) +} + func (ndb *nodeDB) nodeKey(hash []byte) []byte { return nodeKeyFormat.KeyBytes(hash) } From 86e8371ff984aebbae5ff026a335018a1d1c4841 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 12 Jul 2019 11:44:56 -0700 Subject: [PATCH 11/57] update tree.versions after pruning --- mutable_tree.go | 11 ++++++----- nodedb.go | 14 +++++++++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index 4043ee596..35c3cb9bd 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -397,9 +397,7 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { } else { debug("SAVE TREE %v\n", version) // Save the current tree. - // For now: flushToDisk is calculated in here. Ideally mutableTree has no notion of pruning - flushToDisk := version%tree.ndb.keepEvery == 0 - tree.ndb.SaveBranch(tree.root, flushToDisk) + tree.ndb.SaveTree(tree.root, version) // Assume orphans not needed any more. So don't save any tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveRoot(tree.root, version) @@ -413,8 +411,11 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { tree.lastSaved = tree.ImmutableTree.clone() tree.orphans = map[string]int64{} - // Prune nodeDB - tree.ndb.PruneRecentVersions() + // Prune nodeDB and delete any pruned versions from tree.versions + prunedVersions := tree.ndb.PruneRecentVersions() + for _, pVer := range prunedVersions { + delete(tree.versions, version) + } return tree.Hash(), version, nil } diff --git a/nodedb.go b/nodedb.go index a3056590a..68ce940a1 100644 --- a/nodedb.go +++ b/nodedb.go @@ -154,6 +154,12 @@ func (ndb *nodeDB) Has(hash []byte) bool { return ndb.snapshotDB.Get(key) != nil } +// SaveTree takes a rootNode and version. Saves all nodes in tree using SaveBranch +func (ndb *nodeDB) SaveTree(root *Node, version int64) []byte { + flushToDisk := isSnapshotVersion(version) + return SaveBranch(root, flushToDisk) +} + // SaveBranch saves the given node and all of its descendants. // NOTE: This function clears leftNode/rigthNode recursively and // calls _hash() on the given node. @@ -281,11 +287,13 @@ func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk b } } -func (ndb *nodeDB) PruneRecentVersions() { +func (ndb *nodeDB) PruneRecentVersions() (prunedVersions []int64) { if ndb.latestVersion - ndb.keepRecent <= 0 { - return + return nil } - ndb.DeleteVersionFromRecent(ndb.latestVersion-keepRecent) + pruneVer := ndb.latestVersion-keepRecent + ndb.DeleteVersionFromRecent(pruneVer) + return append(prunedVersions, pruneVer) } func (ndb *nodeDB) nodeKey(hash []byte) []byte { From 6b3597a16c2a42f1be72cb3beacfe0cf448d1d62 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 12 Jul 2019 12:32:08 -0700 Subject: [PATCH 12/57] fix most build errs and allow memDB to be disabled --- go.mod | 22 +++++++++++++ go.sum | 27 +++++++++++++++ mutable_tree.go | 2 +- nodedb.go | 87 ++++++++++++++++++++++++++----------------------- 4 files changed, 96 insertions(+), 42 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..f29923ccb --- /dev/null +++ b/go.mod @@ -0,0 +1,22 @@ +module github.com/tendermint/iavl + +go 1.12 + +require ( + github.com/davecgh/go-spew v1.1.1 + github.com/go-kit/kit v0.6.0 + github.com/go-logfmt/logfmt v0.4.0 + github.com/go-stack/stack v1.8.0 + github.com/gogo/protobuf v1.1.1 + github.com/golang/protobuf v1.1.0 + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db + github.com/jmhodges/levigo v0.0.0-20161115193449-c42d9e0ca023 + github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 + github.com/pkg/errors v0.8.0 + github.com/pmezard/go-difflib v1.0.0 + github.com/stretchr/testify v1.2.2 + github.com/syndtr/goleveldb v0.0.0-20181105012736-f9080354173f + github.com/tendermint/go-amino v0.14.1 + github.com/tendermint/tendermint v0.30.2 + golang.org/x/crypto v0.0.0-20181126093934-9eb0be3963ea +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..fb95be666 --- /dev/null +++ b/go.sum @@ -0,0 +1,27 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.6.0 h1:wTifptAGIyIuir4bRyN4h7+kAa2a4eepLYVmRe5qqQ8= +github.com/go-kit/kit v0.6.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/jmhodges/levigo v0.0.0-20161115193449-c42d9e0ca023/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/syndtr/goleveldb v0.0.0-20181105012736-f9080354173f h1:EEVjSRihF8NIbfyCcErpSpNHEKrY3s8EAwqiPENZZn8= +github.com/syndtr/goleveldb v0.0.0-20181105012736-f9080354173f/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/tendermint/go-amino v0.14.1 h1:o2WudxNfdLNBwMyl2dqOJxiro5rfrEaU0Ugs6offJMk= +github.com/tendermint/go-amino v0.14.1/go.mod h1:i/UKE5Uocn+argJJBb12qTZsCDBcAYMbR92AaJVmKso= +github.com/tendermint/tendermint v0.30.2 h1:FpaNQABxRJedjayiX+X6G8mo6FWNnD2eWcKunv3+a60= +github.com/tendermint/tendermint v0.30.2/go.mod h1:ymcPyWblXCplCPQjbOYbrF1fWnpslATMVqiGgWbZrlc= +golang.org/x/crypto v0.0.0-20181126093934-9eb0be3963ea/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/mutable_tree.go b/mutable_tree.go index 35c3cb9bd..7acac8835 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -433,7 +433,7 @@ func (tree *MutableTree) DeleteVersion(version int64) error { return cmn.ErrorWrap(ErrVersionDoesNotExist, "") } - tree.ndb.DeleteVersion(version, tree.latestVersion) + tree.ndb.DeleteVersion(version, true) tree.ndb.Commit() delete(tree.versions, version) diff --git a/nodedb.go b/nodedb.go index 68ce940a1..6ccd91f3e 100644 --- a/nodedb.go +++ b/nodedb.go @@ -47,7 +47,7 @@ type nodeDB struct { nodeCacheQueue *list.List // LRU queue of cache elements. Used for deletion. } -func newNodeDB(snapshotDB dbm.DB, recentDB dmb.DB, cacheSize int, keepRecent, keepEvery int64) *nodeDB { +func newNodeDB(snapshotDB dbm.DB, recentDB dbm.DB, cacheSize int, keepRecent, keepEvery int64) *nodeDB { ndb := &nodeDB{ snapshotDB: snapshotDB, recentDB: recentDB, @@ -63,6 +63,14 @@ func newNodeDB(snapshotDB dbm.DB, recentDB dmb.DB, cacheSize int, keepRecent, ke return ndb } +func (ndb *nodeDB) isSnapshotVersion(version int64) bool { + return version%ndb.keepEvery == 0 +} + +func (ndb *nodeDB) isRecentVersion(version int64) bool { + return ndb.keepRecent != 0 && version > ndb.latestVersion-ndb.keepRecent +} + // GetNode gets a node from memory or disk. If it is an inner node, it does not // load its children. func (ndb *nodeDB) GetNode(hash []byte) *Node { @@ -92,7 +100,7 @@ func (ndb *nodeDB) GetNode(hash []byte) *Node { persisted = true } - node, err = MakeNode(buf) + node, err := MakeNode(buf) if err != nil { panic(fmt.Sprintf("Error reading Node. bytes: %x, error: %v", buf, err)) } @@ -145,7 +153,7 @@ func (ndb *nodeDB) Has(hash []byte) bool { } if ldb, ok := ndb.snapshotDB.(*dbm.GoLevelDB); ok { - exists, err = ldb.DB().Has(key, nil) + exists, err := ldb.DB().Has(key, nil) if err != nil { panic("Got error from leveldb: " + err.Error()) } @@ -156,8 +164,8 @@ func (ndb *nodeDB) Has(hash []byte) bool { // SaveTree takes a rootNode and version. Saves all nodes in tree using SaveBranch func (ndb *nodeDB) SaveTree(root *Node, version int64) []byte { - flushToDisk := isSnapshotVersion(version) - return SaveBranch(root, flushToDisk) + flushToDisk := ndb.isSnapshotVersion(version) + return ndb.SaveBranch(root, flushToDisk) } // SaveBranch saves the given node and all of its descendants. @@ -187,11 +195,11 @@ func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { // DeleteVersion deletes a tree version from disk. func (ndb *nodeDB) DeleteVersion(version int64, checkLatestVersion bool) { - deleteVersion(version, checkLatestVersion, false) + ndb.deleteVersion(version, checkLatestVersion, false) } func (ndb *nodeDB) DeleteVersionFromRecent(version int64, checkLatestVersion bool) { - deleteVersion(version, checkLatestVersion, true) + ndb.deleteVersion(version, checkLatestVersion, true) } func (ndb *nodeDB) deleteVersion(version int64, checkLatestVersion, memOnly bool) { @@ -209,12 +217,12 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { ndb.mtx.Lock() defer ndb.mtx.Unlock() - toVersion := ndb.getPreviousVersionFromDB(version, ndb.recentDB) + toVersion := ndb.getPreviousVersion(version) for hash, fromVersion := range orphans { debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) // if snapshot version in between fromVersion and toVersion, then flush to disk - flushToDisk := containsSnapshot(fromVersion, toVersion) + flushToDisk := fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery ndb.saveOrphan([]byte(hash), fromVersion, toVersion, flushToDisk) } } @@ -224,12 +232,14 @@ func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushTo if fromVersion > toVersion { panic(fmt.Sprintf("Orphan expires before it comes alive. %d > %d", fromVersion, toVersion)) } - key := ndb.orphanKey(fromVersion, toVersion, hash) - ndb.recentBatch.Set(key, hash) + if ndb.isRecentVersion(toVersion) { + key := ndb.orphanKey(fromVersion, toVersion, hash) + ndb.recentBatch.Set(key, hash) + } if flushToDisk { // save to disk with toVersion equal to snapshotVersion closest to original toVersion - snapVersion := toVersion - (toVersion%keepEvery) - key = ndb.orphanKey(fromVersion, snapVersion, hash) + snapVersion := toVersion - (toVersion % ndb.keepEvery) + key := ndb.orphanKey(fromVersion, snapVersion, hash) ndb.snapshotBatch.Set(key, hash) } } @@ -237,17 +247,20 @@ func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushTo // deleteOrphans deletes orphaned nodes from disk, and the associated orphan // entries. func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { - deleteOrphansMem(version) - if isSnapshotVersion(version) && !memOnly { + ndb.deleteOrphansMem(version) + if ndb.isSnapshotVersion(version) && !memOnly { traverseOrphansVersionFromDB(ndb.snapshotDB, version, func(key, hash []byte) { ndb.snapshotDB.Delete(key) - deleteOrphansHelper(ndb.snapshotDB, ndb.snapshotBatch, true, key, hash) + ndb.deleteOrphansHelper(ndb.snapshotDB, ndb.snapshotBatch, true, key, hash) }) } } func (ndb *nodeDB) deleteOrphansMem(version int64) { traverseOrphansVersionFromDB(ndb.recentDB, version, func(key, hash []byte) { + if ndb.keepRecent == 0 { + return + } ndb.recentBatch.Delete(key) // common case, we are deleting orphans from least recent version that is getting pruned from memDB if version == ndb.latestVersion-ndb.keepRecent { @@ -256,7 +269,7 @@ func (ndb *nodeDB) deleteOrphansMem(version int64) { ndb.uncacheNode(hash) return } - + // user is manually deleting version from memDB // thus predecessor may exist in memDB // Will be zero if there is no previous version. @@ -271,7 +284,7 @@ func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk b // `toVersion` are always equal. orphanKeyFormat.Scan(key, &toVersion, &fromVersion) - predecessor := getPreviousVersionFromDB(toVersion, ndb.recentDB) + predecessor := getPreviousVersionFromDB(toVersion, db) // If there is no predecessor, or the predecessor is earlier than the // beginning of the lifetime (ie: negative lifetime), or the lifetime // spans a single version and that version is the one being deleted, we @@ -288,11 +301,11 @@ func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk b } func (ndb *nodeDB) PruneRecentVersions() (prunedVersions []int64) { - if ndb.latestVersion - ndb.keepRecent <= 0 { + if ndb.keepRecent == 0 || ndb.latestVersion-ndb.keepRecent <= 0 { return nil } - pruneVer := ndb.latestVersion-keepRecent - ndb.DeleteVersionFromRecent(pruneVer) + pruneVer := ndb.latestVersion - ndb.keepRecent + ndb.DeleteVersionFromRecent(pruneVer, true) return append(prunedVersions, pruneVer) } @@ -327,16 +340,16 @@ func (ndb *nodeDB) resetLatestVersion(version int64) { func (ndb *nodeDB) getPreviousVersion(version int64) int64 { // If version exists in recentDB, check recentDB for any previous version - if version > ndb.latestVersion-ndb.keepRecent { + if ndb.isRecentVersion(version) { prev := getPreviousVersionFromDB(version, ndb.recentDB) - if prev { + if prev != 0 { return prev } } - return ndb.getPreviousVersionFromDB(version, ndb.snapshotDB) + return getPreviousVersionFromDB(version, ndb.snapshotDB) } -func (ndb *nodeDB) getPreviousVersionFromDB(version int64, db dbm.DB) int64 { +func getPreviousVersionFromDB(version int64, db dbm.DB) int64 { itr := db.ReverseIterator( rootKeyFormat.Key(1), rootKeyFormat.Key(version), @@ -361,7 +374,7 @@ func (ndb *nodeDB) deleteRoot(version int64, checkLatestVersion, memOnly bool) { key := ndb.rootKey(version) ndb.recentBatch.Delete(key) - if isSnapshotVersion(version) && !memOnly { + if ndb.isSnapshotVersion(version) && !memOnly { ndb.snapshotBatch.Delete(key) } } @@ -375,10 +388,10 @@ func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { // traverse will return the node twice. func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { prefix := orphanKeyFormat.Key(version) - if version > ndb.latestVersion-ndb.keepRecent { + if ndb.isRecentVersion(version) { traversePrefixFromDB(ndb.recentDB, prefix, fn) } - if version%keepEvery == 0 { + if ndb.isSnapshotVersion(version) { traversePrefixFromDB(ndb.snapshotDB, prefix, fn) } } @@ -424,7 +437,7 @@ func (ndb *nodeDB) traversePrefix(prefix []byte, fn func(k, v []byte)) { fn(memItr.Key(), memItr.Value()) } - itr := dbm.IteratePrefix(ndb.db, prefix) + itr := dbm.IteratePrefix(ndb.snapshotDB, prefix) defer itr.Close() for ; itr.Valid(); itr.Next() { @@ -472,11 +485,11 @@ func (ndb *nodeDB) Commit() { ndb.recentBatch.Write() ndb.recentBatch.Close() ndb.snapshotBatch = ndb.snapshotDB.NewBatch() - ndb.recentBatch = ndb.dbMem.NewBatch() + ndb.recentBatch = ndb.recentDB.NewBatch() } func (ndb *nodeDB) getRoot(version int64) []byte { - if isRecentVersion(version) { + if ndb.isRecentVersion(version) { memroot := ndb.recentDB.Get(ndb.rootKey(version)) // TODO: maybe I shouldn't check in snapshot if it isn't here if len(memroot) > 0 { @@ -505,12 +518,12 @@ func (ndb *nodeDB) SaveRoot(root *Node, version int64) error { if len(root.hash) == 0 { panic("Hash should not be empty") } - return ndb.saveRoot(root.hash, version, isSnapshotVersion(version)) + return ndb.saveRoot(root.hash, version, ndb.isSnapshotVersion(version)) } // SaveEmptyRoot creates an entry on disk for an empty root. func (ndb *nodeDB) SaveEmptyRoot(version int64) error { - return ndb.saveRoot([]byte{}, version, isSnapshotVersion(version)) + return ndb.saveRoot([]byte{}, version, ndb.isSnapshotVersion(version)) } func (ndb *nodeDB) saveRoot(hash []byte, version int64, flushToDisk bool) error { @@ -533,14 +546,6 @@ func (ndb *nodeDB) saveRoot(hash []byte, version int64, flushToDisk bool) error ////////////////// Utility and test functions ///////////////////////////////// -func (ndb *nodeDB) isSnapshotVersion(version int64) bool { - return toVersion%ndb.keepEvery == 0 -} - -func (ndb *nodeDB) isRecentVersion(version int64) bool { - return version > ndb.latestVersion-ndb.keepRecent -} - func (ndb *nodeDB) leafNodes() []*Node { leaves := []*Node{} From bb45bf12a002a26084a49beae07fd6afa80cfe50 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 12 Jul 2019 12:52:54 -0700 Subject: [PATCH 13/57] fix build errors and maintain backward compatibility --- immutable_tree.go | 10 +++++++++- mutable_tree.go | 15 ++++++++++----- nodedb.go | 4 ++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/immutable_tree.go b/immutable_tree.go index 57573ea8a..10d305450 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -24,7 +24,15 @@ func NewImmutableTree(db dbm.DB, cacheSize int) *ImmutableTree { } return &ImmutableTree{ // NodeDB-backed Tree. - ndb: newNodeDB(db, cacheSize), + // memDB created but should never be written to + ndb: newNodeDB(db, dbm.NewMemDB(), cacheSize, 1, 0), + } +} + +func NewImmutableTreePruning(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *ImmutableTree { + return &ImmutableTree{ + // NodeDB-backed Tree. + ndb: newNodeDB(snapDB, recentDB, cacheSize, keepEvery, keepRecent), } } diff --git a/mutable_tree.go b/mutable_tree.go index 6d15c0d0d..acb104250 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -22,8 +22,15 @@ type MutableTree struct { } // NewMutableTree returns a new tree with the specified cache size and datastore and pruning options -func NewMutableTree(db dbm.DB, cacheSize int, keepEvery, keepRecent int64) *MutableTree { - ndb := newNodeDB(db, cacheSize, keepRecent, keepEvery) +// To maintain backwards compatibility, this function will initialize PruningStrategy{keepEvery: 1, keepRecent: 0} +func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { + // memDB is initialized but should never be written to + memDB := dbm.NewMemDB() + return NewMutableTreePruningOpts(db, memDB, cacheSize, 1, 0) +} + +func NewMutableTreePruningOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *MutableTree { + ndb := newNodeDB(snapDB, recentDB, cacheSize, keepEvery, keepRecent) head := &ImmutableTree{ndb: ndb} return &MutableTree{ @@ -404,14 +411,12 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { // There can still be orphans, for example if the root is the node being // removed. debug("SAVE EMPTY TREE %v\n", version) - // Assume orphans not needed any more. So don't save any tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveEmptyRoot(version) } else { debug("SAVE TREE %v\n", version) // Save the current tree. tree.ndb.SaveTree(tree.root, version) - // Assume orphans not needed any more. So don't save any tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveRoot(tree.root, version) } @@ -427,7 +432,7 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { // Prune nodeDB and delete any pruned versions from tree.versions prunedVersions := tree.ndb.PruneRecentVersions() for _, pVer := range prunedVersions { - delete(tree.versions, version) + delete(tree.versions, pVer) } return tree.Hash(), version, nil diff --git a/nodedb.go b/nodedb.go index 6ccd91f3e..9228c775d 100644 --- a/nodedb.go +++ b/nodedb.go @@ -47,7 +47,7 @@ type nodeDB struct { nodeCacheQueue *list.List // LRU queue of cache elements. Used for deletion. } -func newNodeDB(snapshotDB dbm.DB, recentDB dbm.DB, cacheSize int, keepRecent, keepEvery int64) *nodeDB { +func newNodeDB(snapshotDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *nodeDB { ndb := &nodeDB{ snapshotDB: snapshotDB, recentDB: recentDB, @@ -64,7 +64,7 @@ func newNodeDB(snapshotDB dbm.DB, recentDB dbm.DB, cacheSize int, keepRecent, ke } func (ndb *nodeDB) isSnapshotVersion(version int64) bool { - return version%ndb.keepEvery == 0 + return ndb.keepEvery != 0 && version%ndb.keepEvery == 0 } func (ndb *nodeDB) isRecentVersion(version int64) bool { From bb129a9cb1bf8c7dbc43118009c308af165fd1ff Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 12 Jul 2019 12:55:08 -0700 Subject: [PATCH 14/57] only write to db if pruning opts allow it --- nodedb.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nodedb.go b/nodedb.go index 9228c775d..c6bd922ab 100644 --- a/nodedb.go +++ b/nodedb.go @@ -480,10 +480,14 @@ func (ndb *nodeDB) Commit() { ndb.mtx.Lock() defer ndb.mtx.Unlock() - ndb.snapshotBatch.Write() - ndb.snapshotBatch.Close() - ndb.recentBatch.Write() - ndb.recentBatch.Close() + if ndb.keepEvery != 0 { + ndb.snapshotBatch.Write() + ndb.snapshotBatch.Close() + } + if ndb.keepRecent != 0 { + ndb.recentBatch.Write() + ndb.recentBatch.Close() + } ndb.snapshotBatch = ndb.snapshotDB.NewBatch() ndb.recentBatch = ndb.recentDB.NewBatch() } From f01c63ac861a68ea9671f7180ed888524fcd9c10 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 12 Jul 2019 15:45:31 -0700 Subject: [PATCH 15/57] debugging... --- immutable_tree.go | 2 +- logger.go | 2 +- mutable_tree.go | 12 +++++++----- nodedb.go | 10 ++++++---- tree_test.go | 5 +++-- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/immutable_tree.go b/immutable_tree.go index 10d305450..194daf987 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -29,7 +29,7 @@ func NewImmutableTree(db dbm.DB, cacheSize int) *ImmutableTree { } } -func NewImmutableTreePruning(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *ImmutableTree { +func NewImmutableTreePruningOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *ImmutableTree { return &ImmutableTree{ // NodeDB-backed Tree. ndb: newNodeDB(snapDB, recentDB, cacheSize, keepEvery, keepRecent), diff --git a/logger.go b/logger.go index b53ce34ed..087e1e4fb 100644 --- a/logger.go +++ b/logger.go @@ -5,7 +5,7 @@ import ( ) func debug(format string, args ...interface{}) { - if false { + if true { fmt.Printf(format, args...) } } diff --git a/mutable_tree.go b/mutable_tree.go index acb104250..2908858eb 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -420,6 +420,13 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveRoot(tree.root, version) } + + // Prune nodeDB and delete any pruned versions from tree.versions + prunedVersions := tree.ndb.PruneRecentVersions() + for _, pVer := range prunedVersions { + delete(tree.versions, pVer) + } + tree.ndb.Commit() tree.version = version tree.versions[version] = true @@ -429,11 +436,6 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { tree.lastSaved = tree.ImmutableTree.clone() tree.orphans = map[string]int64{} - // Prune nodeDB and delete any pruned versions from tree.versions - prunedVersions := tree.ndb.PruneRecentVersions() - for _, pVer := range prunedVersions { - delete(tree.versions, pVer) - } return tree.Hash(), version, nil } diff --git a/nodedb.go b/nodedb.go index c6bd922ab..4181853e6 100644 --- a/nodedb.go +++ b/nodedb.go @@ -93,7 +93,7 @@ func (ndb *nodeDB) GetNode(hash []byte) *Node { persisted := false if buf == nil { // Doesn't exist, load from disk - buf := ndb.snapshotDB.Get(ndb.nodeKey(hash)) + buf = ndb.snapshotDB.Get(ndb.nodeKey(hash)) if buf == nil { panic(fmt.Sprintf("Value missing for hash %x corresponding to nodeKey %s", hash, ndb.nodeKey(hash))) } @@ -221,8 +221,8 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { for hash, fromVersion := range orphans { debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) - // if snapshot version in between fromVersion and toVersion, then flush to disk - flushToDisk := fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery + // if snapshot version in between fromVersion and toVersion, then flush to disk. Or fromVersion == toVersion == snapshotVersion + flushToDisk := fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery || (ndb.isSnapshotVersion(toVersion) && ndb.isSnapshotVersion(fromVersion)) ndb.saveOrphan([]byte(hash), fromVersion, toVersion, flushToDisk) } } @@ -247,7 +247,9 @@ func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushTo // deleteOrphans deletes orphaned nodes from disk, and the associated orphan // entries. func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { - ndb.deleteOrphansMem(version) + if ndb.keepRecent != 0 { + ndb.deleteOrphansMem(version) + } if ndb.isSnapshotVersion(version) && !memOnly { traverseOrphansVersionFromDB(ndb.snapshotDB, version, func(key, hash []byte) { ndb.snapshotDB.Delete(key) diff --git a/tree_test.go b/tree_test.go index 32826a25e..a819c3342 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1057,8 +1057,9 @@ func TestOrphans(t *testing.T) { } idx := mathrand.Perm(NUMVERSIONS - 2) - for i := range idx { - err := tree.DeleteVersion(int64(i + 2)) + fmt.Printf("PERMUTATION: %v\n", idx) + for _, v := range idx { + err := tree.DeleteVersion(int64(v + 1)) require.NoError(err, "DeleteVersion should not error") } From 78060fee6af65db5fbdd0faa4f6881e3ef557d69 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 12 Jul 2019 15:51:23 -0700 Subject: [PATCH 16/57] remove print statement --- tree_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tree_test.go b/tree_test.go index a819c3342..20bc34f58 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1057,7 +1057,6 @@ func TestOrphans(t *testing.T) { } idx := mathrand.Perm(NUMVERSIONS - 2) - fmt.Printf("PERMUTATION: %v\n", idx) for _, v := range idx { err := tree.DeleteVersion(int64(v + 1)) require.NoError(err, "DeleteVersion should not error") From 2f3502a7372c379fb5c0f7931b3cf70a8c2a3f8c Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 12 Jul 2019 16:29:53 -0700 Subject: [PATCH 17/57] optimize deleteOrphans and fix final test errors --- benchmarks/bench_test.go | 21 ++------------------- logger.go | 2 +- nodedb.go | 10 ++++++---- tree_test.go | 6 +++--- 4 files changed, 12 insertions(+), 27 deletions(-) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 4b7458e82..855fb3b54 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -36,32 +36,15 @@ func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.Mutab return t, keys } -// commit tree saves a new version and deletes and old one... +// commit tree saves a new version according to pruning strategy passed into IAVL func commitTree(b *testing.B, t *iavl.MutableTree) { t.Hash() - var err error - var version int64 - _, version, err = t.SaveVersionMem() //this will flush for us every so often + _, _, err := t.SaveVersion() //this will flush for us every so often if err != nil { b.Errorf("Can't save: %v", err) } - if version > historySize { - err = t.DeleteVersionFull(version-historySize, false) - if err != nil { - b.Errorf("Can't delete: %v", err) - } - } - - //Lets flush every X blocks - if version%historySize == 0 { - //We don't need to delete all the versions when using mem versions, just flush to disk - _, _, err = t.FlushMemVersionDisk() - if err != nil { - b.Errorf("Can't save: %v", err) - } - } } func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) { diff --git a/logger.go b/logger.go index 087e1e4fb..b53ce34ed 100644 --- a/logger.go +++ b/logger.go @@ -5,7 +5,7 @@ import ( ) func debug(format string, args ...interface{}) { - if true { + if false { fmt.Printf(format, args...) } } diff --git a/nodedb.go b/nodedb.go index 4181853e6..be19b67f5 100644 --- a/nodedb.go +++ b/nodedb.go @@ -251,9 +251,10 @@ func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { ndb.deleteOrphansMem(version) } if ndb.isSnapshotVersion(version) && !memOnly { + predecessor := getPreviousVersionFromDB(version, ndb.snapshotDB) traverseOrphansVersionFromDB(ndb.snapshotDB, version, func(key, hash []byte) { ndb.snapshotDB.Delete(key) - ndb.deleteOrphansHelper(ndb.snapshotDB, ndb.snapshotBatch, true, key, hash) + ndb.deleteOrphansHelper(ndb.snapshotDB, ndb.snapshotBatch, true, predecessor, key, hash) }) } } @@ -272,21 +273,22 @@ func (ndb *nodeDB) deleteOrphansMem(version int64) { return } + predecessor := getPreviousVersionFromDB(version, ndb.recentDB) + // user is manually deleting version from memDB // thus predecessor may exist in memDB // Will be zero if there is no previous version. - ndb.deleteOrphansHelper(ndb.recentDB, ndb.recentBatch, false, key, hash) + ndb.deleteOrphansHelper(ndb.recentDB, ndb.recentBatch, false, predecessor, key, hash) }) } -func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk bool, key, hash []byte) { +func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk bool, predecessor int64, key, hash []byte) { var fromVersion, toVersion int64 // See comment on `orphanKeyFmt`. Note that here, `version` and // `toVersion` are always equal. orphanKeyFormat.Scan(key, &toVersion, &fromVersion) - predecessor := getPreviousVersionFromDB(toVersion, db) // If there is no predecessor, or the predecessor is earlier than the // beginning of the lifetime (ie: negative lifetime), or the lifetime // spans a single version and that version is the one being deleted, we diff --git a/tree_test.go b/tree_test.go index 20bc34f58..72ace5dde 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1041,7 +1041,7 @@ func TestVersionedTreeProofs(t *testing.T) { func TestOrphans(t *testing.T) { //If you create a sequence of saved versions //Then randomly delete versions other than the first and last until only those two remain - //Any remaining orphan nodes should be constrained to just the first version + //Any remaining orphan nodes should either have fromVersion == firstVersion || toVersion == lastVersion require := require.New(t) tree := NewMutableTree(db.NewMemDB(), 100) @@ -1065,8 +1065,8 @@ func TestOrphans(t *testing.T) { tree.ndb.traverseOrphans(func(k, v []byte) { var fromVersion, toVersion int64 orphanKeyFormat.Scan(k, &toVersion, &fromVersion) - require.Equal(fromVersion, int64(1), "fromVersion should be 1") - require.Equal(toVersion, int64(1), "toVersion should be 1") + require.True(fromVersion == int64(1) || toVersion == int64(99), fmt.Sprintf(`Unexpected orphan key exists: %v with fromVersion = %d and toVersion = %d.\n + Any orphan remaining in db should have either fromVersion == 1 or toVersion == 99. Since Version 1 and 99 are only versions in db`, k, fromVersion, toVersion)) }) } From bce37e31d28415d5269db1be4bb55d43615d44e6 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 12 Jul 2019 16:35:53 -0700 Subject: [PATCH 18/57] lint --- benchmarks/bench_test.go | 2 -- mutable_tree.go | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 855fb3b54..49c6f89a4 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -11,8 +11,6 @@ import ( db "github.com/tendermint/tendermint/libs/db" ) -const historySize = 20 - func randBytes(length int) []byte { key := make([]byte, length) // math.rand.Read always returns err=nil diff --git a/mutable_tree.go b/mutable_tree.go index 2908858eb..2c195d2ce 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -426,7 +426,7 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { for _, pVer := range prunedVersions { delete(tree.versions, pVer) } - + tree.ndb.Commit() tree.version = version tree.versions[version] = true @@ -436,7 +436,6 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { tree.lastSaved = tree.ImmutableTree.clone() tree.orphans = map[string]int64{} - return tree.Hash(), version, nil } From 46d16a010f7b9136a31d29e9d345a36029891373 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 15 Jul 2019 09:05:28 -0700 Subject: [PATCH 19/57] add brief design doc and small fix --- PRUNING.md | 34 ++++++++++++++++++++++++++++++++++ nodedb.go | 5 ++++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 PRUNING.md diff --git a/PRUNING.md b/PRUNING.md new file mode 100644 index 000000000..6c8013cd5 --- /dev/null +++ b/PRUNING.md @@ -0,0 +1,34 @@ +Setting Pruning fields in the IAVL tree can optimize performance by only writing versions to disk if they are meant to be persisted indefinitely. Versions that are known to be deleted eventually are temporarily held in memory until they are ready to be pruned. This greatly reduces the I/O load of IAVL. + +We can set custom pruning fields in IAVL using: `NewMutableTreePruningOpts` + + +## Current design + +### NodeDB +NodeDB has extra fields: +``` +recentDB dbm.DB // Memory node storage. +recentBatch dbm.Batch // Batched writing buffer for memDB. + +// Pruning fields +keepEvery int64n // Saves version to disk periodically +keepRecent int64 // Saves recent versions in memory +``` + +If version is not going to be persisted to disk, the version is simply saved in `recentDB` (typically a memDB) +If version is persisted to disk, the version is written to `recentDB` **and** `snapshotDB` (typically `levelDB`) + +#### Orphans: +Save orphan to memDB under `o|toVersion|fromVersion`. + +If there exists snapshot version `snapVersion` s.t. `fromVersion < snapVersion < toVersion`, save orphan to disk as well under `o|snapVersion|fromVersion`. +NOTE: in unlikely event, that two snapshot versions exist between `fromVersion` and `toVersion`, we use closest snapshot version that is less than `toVersion` + +Can then simply use the old delete algorithm with some minor simplifications/optimizations + +### MutableTree + +MutableTree can be instantiated with a pruning-aware NodeDB. + +When `MutableTree` saves a new Version, it also calls `PruneRecentVersions` on nodeDB which causes oldest version in recentDB (`latestVersion - keepRecent`) to get pruned. \ No newline at end of file diff --git a/nodedb.go b/nodedb.go index be19b67f5..412ed0728 100644 --- a/nodedb.go +++ b/nodedb.go @@ -173,7 +173,10 @@ func (ndb *nodeDB) SaveTree(root *Node, version int64) []byte { // calls _hash() on the given node. // TODO refactor, maybe use hashWithCount() but provide a callback. func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { - if node.saved { + if node.saved && !flushToDisk { + return node.hash + } + if node.persisted { return node.hash } From d244ed382e9fdefd25cd8b9825af4e80e563d1ef Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 15 Jul 2019 15:50:01 -0700 Subject: [PATCH 20/57] add basic pruning tests --- nodedb.go | 8 ++- pruning_test.go | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 pruning_test.go diff --git a/nodedb.go b/nodedb.go index 412ed0728..7e034f2c6 100644 --- a/nodedb.go +++ b/nodedb.go @@ -313,6 +313,9 @@ func (ndb *nodeDB) PruneRecentVersions() (prunedVersions []int64) { } pruneVer := ndb.latestVersion - ndb.keepRecent ndb.DeleteVersionFromRecent(pruneVer, true) + if ndb.isSnapshotVersion(pruneVer) { + return nil + } return append(prunedVersions, pruneVer) } @@ -390,6 +393,10 @@ func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { ndb.traversePrefix(orphanKeyFormat.Key(), fn) } +func traverseOrphansFromDB(db dbm.DB, fn func(k, v []byte)) { + traversePrefixFromDB(db, orphanKeyFormat.Key(), fn) +} + // Traverse orphans ending at a certain version. // NOTE: If orphan is in recentDB and levelDB (version > latestVersion-keepRecent && version%keepEvery == 0) // traverse will return the node twice. @@ -511,7 +518,6 @@ func (ndb *nodeDB) getRoot(version int64) []byte { return ndb.snapshotDB.Get(ndb.rootKey(version)) } -// NOTE: will get duplicated root if snapshot version exists in DB and memDB func (ndb *nodeDB) getRoots() (map[int64][]byte, error) { roots := map[int64][]byte{} diff --git a/pruning_test.go b/pruning_test.go new file mode 100644 index 000000000..37a741046 --- /dev/null +++ b/pruning_test.go @@ -0,0 +1,154 @@ +package iavl + +import ( + "encoding/binary" + "fmt" + "os" + "math/rand" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/db" +) + +func getTestDBs() (db.DB, db.DB, func()) { + d, err := db.NewGoLevelDB("test", ".") + if err != nil { + panic(err) + } + return d, db.NewMemDB(), func() { + d.Close() + os.RemoveAll("./test.db") + } +} + +func TestSave(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB + keepEvery := (rand.Int63n(3) + 1) * 100 + mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + + // create 1000 versions + for i := 0; i < 1000; i++ { + // set 5 keys per version + for j := 0; j < 5; j++ { + key := make([]byte, 8) + val := make([]byte, 8) + binary.BigEndian.PutUint64(key, uint64(rand.Int63())) + binary.BigEndian.PutUint64(val, uint64(rand.Int63())) + mt.Set(key, val) + } + _, _, err := mt.SaveVersion() + require.Nil(t, err, "SaveVersion failed") + } + + versions := mt.AvailableVersions() + // check that all available versions are expected. + for _, v := range versions { + ver := int64(v) + // check that version is supposed to exist given pruning strategy + require.True(t, ver%keepEvery == 0 || mt.Version()-ver <= keepRecent, + fmt.Sprintf("Version: %d should not exist. KeepEvery: %d, KeepRecent: %d", v, keepEvery, keepRecent)) + + // check that root exists in nodeDB + lv, err := mt.LazyLoadVersion(ver) + require.Equal(t, ver, lv, "Version returned by LazyLoadVersion is wrong") + require.Nil(t, err, "Version should exist in nodeDB") + } + + // check all expected versions are available. + for j := keepEvery; j <= mt.Version(); j += keepEvery { + require.True(t, mt.VersionExists(int64(j)), fmt.Sprintf("Expected snapshot version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", j, keepEvery, keepRecent)) + } + for k := mt.Version()-keepRecent+1; k <= mt.Version(); k++ { + require.True(t, mt.VersionExists(int64(k)), fmt.Sprintf("Expected recent version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", k, keepEvery, keepRecent)) + } + + // check that there only exists correct number of roots in nodeDB + roots, err := mt.ndb.getRoots() + require.Nil(t, err, "Error in getRoots") + numRoots := 1000 / keepEvery + keepRecent + // decrement if there is overlap between snapshot and recent versions + if 1000 % keepEvery == 0 { + numRoots-- + } + require.Equal(t, numRoots, int64(len(roots)), "nodeDB does not contain expected number of roots") +} + +func TestDeleteOrphans(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB + keepEvery := (rand.Int63n(3) + 1) * 100 + mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + + // create 1000 versions + for i := 0; i < 1000; i++ { + // set 5 keys per version + for j := 0; j < 5; j++ { + key := make([]byte, 8) + val := make([]byte, 8) + binary.BigEndian.PutUint64(key, uint64(rand.Int63())) + binary.BigEndian.PutUint64(val, uint64(rand.Int63())) + mt.Set(key, val) + } + _, _, err := mt.SaveVersion() + require.Nil(t, err, "SaveVersion failed") + } + + snapfn := func(key, v []byte) { + var fromVersion, toVersion int64 + + // See comment on `orphanKeyFmt`. Note that here, `version` and + // `toVersion` are always equal. + orphanKeyFormat.Scan(key, &toVersion, &fromVersion) + + // toVersion must be snapshotVersion + require.True(t, toVersion%keepEvery == 0, fmt.Sprintf("Orphan in snapshotDB has unexpected toVersion: %d. Should never have been persisted", toVersion)) + } + + // check orphans in snapshotDB are expected + traverseOrphansFromDB(mt.ndb.snapshotDB, snapfn) + + recentFn := func(key, v []byte) { + var fromVersion, toVersion int64 + + // See comment on `orphanKeyFmt`. Note that here, `version` and + // `toVersion` are always equal. + orphanKeyFormat.Scan(key, &toVersion, &fromVersion) + + // toVersion must exist in recentDB + require.True(t, toVersion > mt.Version() - keepRecent, fmt.Sprintf("Orphan in recentDB has unexpected fromVersion: %d. Should have been deleted", fromVersion)) + } + + // check orphans in recentDB are expected + traverseOrphansFromDB(mt.ndb.recentDB, recentFn) + + // delete snapshot orphans except latest version + for j := keepEvery; j < mt.Version(); j += keepEvery { + err := mt.DeleteVersion(j) + require.Nil(t, err, fmt.Sprintf("Could not delete version %d", j)) + } + + lastfn := func(key, v []byte) { + var fromVersion, toVersion int64 + + // See comment on `orphanKeyFmt`. Note that here, `version` and + // `toVersion` are always equal. + orphanKeyFormat.Scan(key, &toVersion, &fromVersion) + + // only orphans that should exist have toVersion = latestVersion + require.Equal(t, mt.Version(), toVersion, fmt.Sprintf("Unexpected Orphan with toVersion: %d", toVersion)) + } + traverseOrphansFromDB(mt.ndb.snapshotDB, lastfn) + + // delete all recent orphans escept latest version + for k := mt.Version()-keepRecent+1; k < mt.Version(); k++ { + err := mt.DeleteVersion(k) + require.Nil(t, err, fmt.Sprintf("Could not delete version %d", k)) + } + traverseOrphansFromDB(mt.ndb.recentDB, lastfn) +} From c5af5b965974d66aacf32c46343a924f078b1392 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Tue, 16 Jul 2019 12:02:30 -0700 Subject: [PATCH 21/57] change tests to use trees with random pruning parameters --- basic_test.go | 15 ++++------ mutable_tree.go | 2 +- testutils_test.go | 12 ++++++-- tree_dotgraph_test.go | 4 +-- tree_fuzz_test.go | 3 +- tree_test.go | 69 +++++++++++++++++++++++-------------------- 6 files changed, 56 insertions(+), 49 deletions(-) diff --git a/basic_test.go b/basic_test.go index 263a04a7a..9c71336b7 100644 --- a/basic_test.go +++ b/basic_test.go @@ -12,7 +12,7 @@ import ( ) func TestBasic(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) up := tree.Set([]byte("1"), []byte("one")) if up { t.Error("Did not expect an update (should have been create)") @@ -185,12 +185,10 @@ func TestUnit(t *testing.T) { } func TestRemove(t *testing.T) { - size := 10000 keyLen, dataLen := 16, 40 - d := db.NewDB("test", "memdb", "") - defer d.Close() - t1 := NewMutableTree(d, size) + size := 10000 + t1 := getTestTree(size) // insert a bunch of random nodes keys := make([][]byte, size) @@ -220,7 +218,7 @@ func TestIntegration(t *testing.T) { } records := make([]*record, 400) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) randomRecord := func() *record { return &record{randstr(20), randstr(20)} @@ -302,7 +300,7 @@ func TestIterateRange(t *testing.T) { } sort.Strings(keys) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) // insert all the data for _, r := range records { @@ -392,8 +390,7 @@ func TestPersistence(t *testing.T) { func TestProof(t *testing.T) { // Construct some random tree - db := db.NewMemDB() - tree := NewMutableTree(db, 100) + tree := getTestTree(100) for i := 0; i < 10; i++ { key, value := randstr(20), randstr(20) tree.Set([]byte(key), []byte(value)) diff --git a/mutable_tree.go b/mutable_tree.go index 2c195d2ce..9f0450633 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -567,7 +567,7 @@ func (tree *MutableTree) balance(node *Node) (newSelf *Node, orphaned []*Node) { func (tree *MutableTree) addOrphans(orphans []*Node) { for _, node := range orphans { - if !node.persisted { + if !node.saved { // We don't need to orphan nodes that were never persisted. continue } diff --git a/testutils_test.go b/testutils_test.go index f802148a4..fcb96e87f 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -28,6 +28,15 @@ func b2i(bz []byte) int { return int(i) } +// Construct a MutableTree with random pruning parameters +func getTestTree(cacheSize int) *MutableTree { + keepRecent := mrand.Int63n(8) + 2 //keep at least 2 versions in memDB + keepEvery := (mrand.Int63n(3) + 1) * 100 // snapshot every {100,200,300} versions + + // Use MemDB for recentDB and snapshotDB + return NewMutableTreePruningOpts(db.NewMemDB(), db.NewMemDB(), cacheSize, keepEvery, keepRecent) +} + // Convenience for a new node func N(l, r interface{}) *Node { var left, right *Node @@ -54,8 +63,7 @@ func N(l, r interface{}) *Node { // Setup a deep node func T(n *Node) *MutableTree { - d := db.NewDB("test", db.MemDBBackend, "") - t := NewMutableTree(d, 0) + t := getTestTree(0) n.hashWithCount() t.root = n diff --git a/tree_dotgraph_test.go b/tree_dotgraph_test.go index cec1475bd..5d6b80047 100644 --- a/tree_dotgraph_test.go +++ b/tree_dotgraph_test.go @@ -3,12 +3,10 @@ package iavl import ( "io/ioutil" "testing" - - "github.com/tendermint/tendermint/libs/db" ) func TestWriteDOTGraph(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, } { diff --git a/tree_fuzz_test.go b/tree_fuzz_test.go index d5e0bcaf9..127496103 100644 --- a/tree_fuzz_test.go +++ b/tree_fuzz_test.go @@ -5,7 +5,6 @@ import ( "testing" cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/libs/db" ) // This file implement fuzz testing by generating programs and then running @@ -110,7 +109,7 @@ func TestMutableTreeFuzz(t *testing.T) { for size := 5; iterations < maxIterations; size++ { for i := 0; i < progsPerIteration/size; i++ { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) program := genRandomProgram(size) err := program.Execute(tree) if err != nil { diff --git a/tree_test.go b/tree_test.go index 72ace5dde..3b7ea545a 100644 --- a/tree_test.go +++ b/tree_test.go @@ -103,7 +103,7 @@ func TestVersionedRandomTreeSmallKeys(t *testing.T) { defer closeDB() tree := NewMutableTree(d, 100) - singleVersionTree := NewMutableTree(db.NewMemDB(), 0) + singleVersionTree := getTestTree(0) versions := 20 keysPerVersion := 50 @@ -144,7 +144,7 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { defer closeDB() tree := NewMutableTree(d, 100) - singleVersionTree := NewMutableTree(db.NewMemDB(), 0) + singleVersionTree := getTestTree(0) versions := 30 keysPerVersion := 50 @@ -180,7 +180,7 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { } func TestVersionedTreeSpecial1(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 100) + tree := getTestTree(100) tree.Set([]byte("C"), []byte("so43QQFN")) tree.SaveVersion() @@ -203,7 +203,7 @@ func TestVersionedTreeSpecial1(t *testing.T) { func TestVersionedRandomTreeSpecial2(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 100) + tree := getTestTree(100) tree.Set([]byte("OFMe2Yvm"), []byte("ez2OtQtE")) tree.Set([]byte("WEN4iN7Y"), []byte("kQNyUalI")) @@ -494,7 +494,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { require.Len(t, tree.ndb.leafNodes(), 3) - tree2 := NewMutableTree(db.NewMemDB(), 0) + tree2 := getTestTree(0) tree2.Set([]byte("key0"), []byte("val2")) tree2.Set([]byte("key2"), []byte("val2")) tree2.Set([]byte("key3"), []byte("val1")) @@ -504,8 +504,7 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { } func TestVersionedTreeOrphanDeleting(t *testing.T) { - mdb := db.NewMemDB() - tree := NewMutableTree(mdb, 0) + tree := getTestTree(0) tree.Set([]byte("key0"), []byte("val0")) tree.Set([]byte("key1"), []byte("val0")) @@ -543,7 +542,7 @@ func TestVersionedTreeOrphanDeleting(t *testing.T) { func TestVersionedTreeSpecialCase(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 100) + tree := getTestTree(100) tree.Set([]byte("key1"), []byte("val0")) tree.Set([]byte("key2"), []byte("val0")) @@ -564,8 +563,8 @@ func TestVersionedTreeSpecialCase(t *testing.T) { func TestVersionedTreeSpecialCase2(t *testing.T) { require := require.New(t) + d := db.NewMemDB() - tree := NewMutableTree(d, 100) tree.Set([]byte("key1"), []byte("val0")) @@ -591,7 +590,7 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { func TestVersionedTreeSpecialCase3(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) tree.Set([]byte("m"), []byte("liWT0U6G")) tree.Set([]byte("G"), []byte("7PxRXwUA")) @@ -670,7 +669,7 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { func TestVersionedTreeErrors(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 100) + tree := getTestTree(100) // Can't delete non-existent versions. require.Error(tree.DeleteVersion(1)) @@ -755,7 +754,7 @@ func TestVersionedCheckpoints(t *testing.T) { func TestVersionedCheckpointsSpecialCase(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) key := []byte("k") tree.Set(key, []byte("val1")) @@ -780,7 +779,7 @@ func TestVersionedCheckpointsSpecialCase(t *testing.T) { } func TestVersionedCheckpointsSpecialCase2(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -800,7 +799,7 @@ func TestVersionedCheckpointsSpecialCase2(t *testing.T) { } func TestVersionedCheckpointsSpecialCase3(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) tree.Set([]byte("n"), []byte("2wUCUs8q")) tree.Set([]byte("l"), []byte("WQ7mvMbc")) @@ -820,7 +819,7 @@ func TestVersionedCheckpointsSpecialCase3(t *testing.T) { } func TestVersionedCheckpointsSpecialCase4(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -852,7 +851,7 @@ func TestVersionedCheckpointsSpecialCase4(t *testing.T) { } func TestVersionedCheckpointsSpecialCase5(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) tree.Set([]byte("R"), []byte("ygZlIzeW")) tree.SaveVersion() @@ -869,7 +868,7 @@ func TestVersionedCheckpointsSpecialCase5(t *testing.T) { } func TestVersionedCheckpointsSpecialCase6(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) tree.Set([]byte("Y"), []byte("MW79JQeV")) tree.Set([]byte("7"), []byte("Kp0ToUJB")) @@ -901,7 +900,7 @@ func TestVersionedCheckpointsSpecialCase6(t *testing.T) { } func TestVersionedCheckpointsSpecialCase7(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 100) + tree := getTestTree(100) tree.Set([]byte("n"), []byte("OtqD3nyn")) tree.Set([]byte("W"), []byte("kMdhJjF5")) @@ -933,9 +932,11 @@ func TestVersionedCheckpointsSpecialCase7(t *testing.T) { tree.GetVersioned([]byte("A"), 3) } +/* +// TODO: Must rewrite to account for pruning func TestVersionedTreeEfficiency(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) versions := 20 keysPerVersion := 100 keysAddedPerVersion := map[int]int{} @@ -950,27 +951,32 @@ func TestVersionedTreeEfficiency(t *testing.T) { tree.SaveVersion() sizeAfter := len(tree.ndb.nodes()) change := sizeAfter - sizeBefore + fmt.Printf("Change after SaveVersion(%d): %d\n", i, change) keysAddedPerVersion[i] = change keysAdded += change } keysDeleted := 0 for i := 1; i < versions; i++ { - sizeBefore := len(tree.ndb.nodes()) - tree.DeleteVersion(int64(i)) - sizeAfter := len(tree.ndb.nodes()) + if tree.VersionExists(int64(i)) { + sizeBefore := len(tree.ndb.nodes()) + tree.DeleteVersion(int64(i)) + sizeAfter := len(tree.ndb.nodes()) - change := sizeBefore - sizeAfter - keysDeleted += change + change := sizeBefore - sizeAfter + fmt.Printf("Change after DeleteVersion(%d): %d\n", i, change) + keysDeleted += change - require.InDelta(change, keysAddedPerVersion[i], float64(keysPerVersion)/5) + // require.InDelta(change, keysAddedPerVersion[i], float64(keysPerVersion)/5) + } } require.Equal(keysAdded-tree.nodeSize(), keysDeleted) } +*/ func TestVersionedTreeProofs(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) tree.Set([]byte("k1"), []byte("v1")) tree.Set([]byte("k2"), []byte("v1")) @@ -1072,7 +1078,7 @@ func TestOrphans(t *testing.T) { func TestVersionedTreeHash(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) require.Nil(tree.Hash()) tree.Set([]byte("I"), []byte("D")) @@ -1094,7 +1100,7 @@ func TestVersionedTreeHash(t *testing.T) { func TestNilValueSemantics(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) require.Panics(func() { tree.Set([]byte("k"), nil) @@ -1104,7 +1110,7 @@ func TestNilValueSemantics(t *testing.T) { func TestCopyValueSemantics(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) val := []byte("v1") @@ -1121,7 +1127,7 @@ func TestCopyValueSemantics(t *testing.T) { func TestRollback(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) tree.Set([]byte("k"), []byte("v")) tree.SaveVersion() @@ -1148,8 +1154,7 @@ func TestRollback(t *testing.T) { } func TestLazyLoadVersion(t *testing.T) { - mdb := db.NewMemDB() - tree := NewMutableTree(mdb, 0) + tree := getTestTree(0) maxVersions := 10 version, err := tree.LazyLoadVersion(0) From 2ab4609b433b2f4caa2f15dc648e56bae328be8f Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Tue, 16 Jul 2019 12:09:09 -0700 Subject: [PATCH 22/57] quick fix --- mutable_tree.go | 2 +- tree_test.go | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index 9f0450633..ae5e1b6c7 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -568,7 +568,7 @@ func (tree *MutableTree) balance(node *Node) (newSelf *Node, orphaned []*Node) { func (tree *MutableTree) addOrphans(orphans []*Node) { for _, node := range orphans { if !node.saved { - // We don't need to orphan nodes that were never persisted. + // We don't need to orphan nodes that were never saved. continue } if len(node.hash) == 0 { diff --git a/tree_test.go b/tree_test.go index 3b7ea545a..56dc125ac 100644 --- a/tree_test.go +++ b/tree_test.go @@ -932,11 +932,9 @@ func TestVersionedCheckpointsSpecialCase7(t *testing.T) { tree.GetVersioned([]byte("A"), 3) } -/* -// TODO: Must rewrite to account for pruning func TestVersionedTreeEfficiency(t *testing.T) { require := require.New(t) - tree := getTestTree(0) + tree := NewMutableTree(db.NewMemDB(), 0) versions := 20 keysPerVersion := 100 keysAddedPerVersion := map[int]int{} @@ -951,7 +949,6 @@ func TestVersionedTreeEfficiency(t *testing.T) { tree.SaveVersion() sizeAfter := len(tree.ndb.nodes()) change := sizeAfter - sizeBefore - fmt.Printf("Change after SaveVersion(%d): %d\n", i, change) keysAddedPerVersion[i] = change keysAdded += change } @@ -964,15 +961,13 @@ func TestVersionedTreeEfficiency(t *testing.T) { sizeAfter := len(tree.ndb.nodes()) change := sizeBefore - sizeAfter - fmt.Printf("Change after DeleteVersion(%d): %d\n", i, change) keysDeleted += change - // require.InDelta(change, keysAddedPerVersion[i], float64(keysPerVersion)/5) + require.InDelta(change, keysAddedPerVersion[i], float64(keysPerVersion)/5) } } require.Equal(keysAdded-tree.nodeSize(), keysDeleted) } -*/ func TestVersionedTreeProofs(t *testing.T) { require := require.New(t) From e50b2979ea9a9a0c48ae1946c1248a913f18af82 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Wed, 17 Jul 2019 10:44:26 -0700 Subject: [PATCH 23/57] add edge case tests --- nodedb.go | 7 +++-- proof_test.go | 7 ++--- pruning_test.go | 79 +++++++++++++++++++++++++++++++++++++++++++++++++ tree_test.go | 4 +-- 4 files changed, 89 insertions(+), 8 deletions(-) diff --git a/nodedb.go b/nodedb.go index 7e034f2c6..aba763cf4 100644 --- a/nodedb.go +++ b/nodedb.go @@ -224,8 +224,11 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { for hash, fromVersion := range orphans { debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) - // if snapshot version in between fromVersion and toVersion, then flush to disk. Or fromVersion == toVersion == snapshotVersion - flushToDisk := fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery || (ndb.isSnapshotVersion(toVersion) && ndb.isSnapshotVersion(fromVersion)) + flushToDisk := false + if ndb.keepEvery != 0 { + // if snapshot version in between fromVersion and toVersion, then flush to disk. Or fromVersion == toVersion == snapshotVersion + flushToDisk = fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery || (ndb.isSnapshotVersion(toVersion) && ndb.isSnapshotVersion(fromVersion)) + } ndb.saveOrphan([]byte(hash), fromVersion, toVersion, flushToDisk) } } diff --git a/proof_test.go b/proof_test.go index 185b8ea76..fb5948d17 100644 --- a/proof_test.go +++ b/proof_test.go @@ -8,12 +8,11 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/go-amino" - "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/test" ) func TestTreeGetWithProof(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) require := require.New(t) for _, ikey := range []byte{0x11, 0x32, 0x50, 0x72, 0x99} { key := []byte{ikey} @@ -47,7 +46,7 @@ func TestTreeGetWithProof(t *testing.T) { } func TestTreeKeyExistsProof(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) root := tree.WorkingHash() // should get false for proof with nil root @@ -113,7 +112,7 @@ func TestTreeKeyExistsProof(t *testing.T) { } func TestTreeKeyInRangeProofs(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree := getTestTree(0) require := require.New(t) keys := []byte{0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7} // 10 total. for _, ikey := range keys { diff --git a/pruning_test.go b/pruning_test.go index 37a741046..cd3c9e9a5 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -152,3 +152,82 @@ func TestDeleteOrphans(t *testing.T) { } traverseOrphansFromDB(mt.ndb.recentDB, lastfn) } + +func TestNoSnapshots(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB + mt := NewMutableTreePruningOpts(db, mdb, 5, 0, keepRecent) // test no snapshots + + for i := 0; i < 50; i++ { + // set 5 keys per version + for j := 0; j < 5; j++ { + key := make([]byte, 8) + val := make([]byte, 8) + binary.BigEndian.PutUint64(key, uint64(rand.Int63())) + binary.BigEndian.PutUint64(val, uint64(rand.Int63())) + mt.Set(key, val) + } + _, _, err := mt.SaveVersion() + require.Nil(t, err, "SaveVersion failed") + } + + versions := mt.AvailableVersions() + require.Equal(t, keepRecent, int64(len(versions)), "Versions in nodeDB not equal to recent versions") + for i := 0; int64(i) < keepRecent; i++ { + seen := false + for _, v := range versions { + if v == int(mt.Version()) - i { + seen = true + } + } + require.True(t, seen, fmt.Sprintf("Version %d is not available even though it is recent", mt.Version() - int64(i))) + } + + size := 0 + traverseFromDB(mt.ndb.snapshotDB, func(k, v []byte) { + size++ + }) + // check that nothing persisted to snapshotDB + require.Equal(t, 0, size, "SnapshotDB should be empty") +} + +func TestNoRecents(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + mt := NewMutableTreePruningOpts(db, mdb, 5, 1, 0) + + for i := 0; i < 50; i++ { + // set 5 keys per version + for j := 0; j < 5; j++ { + key := make([]byte, 8) + val := make([]byte, 8) + binary.BigEndian.PutUint64(key, uint64(rand.Int63())) + binary.BigEndian.PutUint64(val, uint64(rand.Int63())) + mt.Set(key, val) + } + _, _, err := mt.SaveVersion() + require.Nil(t, err, "SaveVersion failed") + } + + size := 0 + traverseFromDB(mt.ndb.recentDB, func(k, v []byte) { + size++ + }) + // check that nothing persisted to recentDB + require.Equal(t, 0, size, "recentDB should be empty") + + versions := mt.AvailableVersions() + require.Equal(t, 50, len(versions), "Versions in nodeDB not equal to snapshot versions") + for i := 1; i <= 50; i++ { + seen := false + for _, v := range versions { + if v == i { + seen = true + } + } + require.True(t, seen, fmt.Sprintf("Version %d is not available even though it is snpashhot version", i)) + } +} diff --git a/tree_test.go b/tree_test.go index 56dc125ac..059addbe7 100644 --- a/tree_test.go +++ b/tree_test.go @@ -819,8 +819,8 @@ func TestVersionedCheckpointsSpecialCase3(t *testing.T) { } func TestVersionedCheckpointsSpecialCase4(t *testing.T) { - tree := getTestTree(0) - + tree := NewMutableTree(db.NewMemDB(), 0) + tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) tree.Set([]byte("H"), []byte("7a9En4uw")) From 0dc07695beb02d04da9e3ce4c53b5fab9e0ea5e2 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Thu, 18 Jul 2019 17:20:30 -0700 Subject: [PATCH 24/57] current progress in testing --- mutable_tree.go | 1 + nodedb.go | 44 +++++++++++++++++++++--- pruning_test.go | 91 ++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 122 insertions(+), 14 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index ae5e1b6c7..100a0b1ab 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -442,6 +442,7 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { // DeleteVersion deletes a tree version from disk. The version can then no // longer be accessed. func (tree *MutableTree) DeleteVersion(version int64) error { + debug("DELETE VERSION: %d\n", version) if version == 0 { return cmn.NewError("version must be greater than 0") } diff --git a/nodedb.go b/nodedb.go index aba763cf4..70bf46304 100644 --- a/nodedb.go +++ b/nodedb.go @@ -190,8 +190,10 @@ func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { node._hash() ndb.SaveNode(node, flushToDisk) - node.leftNode = nil - node.rightNode = nil + if node.persisted { + node.leftNode = nil + node.rightNode = nil + } return node.hash } @@ -223,12 +225,12 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { toVersion := ndb.getPreviousVersion(version) for hash, fromVersion := range orphans { - debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) flushToDisk := false if ndb.keepEvery != 0 { // if snapshot version in between fromVersion and toVersion, then flush to disk. Or fromVersion == toVersion == snapshotVersion flushToDisk = fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery || (ndb.isSnapshotVersion(toVersion) && ndb.isSnapshotVersion(fromVersion)) } + debug("SAVEORPHAN %v-%v %X flushToDisk: %t\n", fromVersion, toVersion, hash, flushToDisk) ndb.saveOrphan([]byte(hash), fromVersion, toVersion, flushToDisk) } } @@ -257,11 +259,13 @@ func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { ndb.deleteOrphansMem(version) } if ndb.isSnapshotVersion(version) && !memOnly { + debug("DELETE ORPHAN FROM SNAPSHOT\n") predecessor := getPreviousVersionFromDB(version, ndb.snapshotDB) traverseOrphansVersionFromDB(ndb.snapshotDB, version, func(key, hash []byte) { ndb.snapshotDB.Delete(key) ndb.deleteOrphansHelper(ndb.snapshotDB, ndb.snapshotBatch, true, predecessor, key, hash) }) + debug("END DELETE ORPHAN FROM SNAPSHOT\n") } } @@ -301,11 +305,11 @@ func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk b // can delete the orphan. Otherwise, we shorten its lifetime, by // moving its endpoint to the previous version. if predecessor < fromVersion || fromVersion == toVersion { - debug("DELETE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) + debug("DELETE predecessor:%v fromVersion:%v toVersion:%v %X flushToDisk: %t\n", predecessor, fromVersion, toVersion, hash, flushToDisk) batch.Delete(ndb.nodeKey(hash)) ndb.uncacheNode(hash) } else { - debug("MOVE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) + debug("MOVE predecessor:%v fromVersion:%v toVersion:%v %X flushToDisk: %t\n", predecessor, fromVersion, toVersion, hash, flushToDisk) ndb.saveOrphan(hash, fromVersion, predecessor, flushToDisk) } } @@ -586,6 +590,15 @@ func (ndb *nodeDB) nodes() []*Node { return nodes } +func (ndb *nodeDB) nodesFromDB(db dbm.DB) []*Node { + nodes := []*Node{} + + ndb.traverseNodesFromDB(db, func(hash []byte, node *Node) { + nodes = append(nodes, node) + }) + return nodes +} + func (ndb *nodeDB) orphans() [][]byte { orphans := [][]byte{} @@ -632,6 +645,27 @@ func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node)) { } } +func (ndb *nodeDB) traverseNodesFromDB(db dbm.DB, fn func(hash []byte, node *Node)) { + nodes := []*Node{} + + traversePrefixFromDB(db, nodeKeyFormat.Key(), func(key, value []byte) { + node, err := MakeNode(value) + if err != nil { + panic(fmt.Sprintf("Couldn't decode node from database: %v", err)) + } + nodeKeyFormat.Scan(key, &node.hash) + nodes = append(nodes, node) + }) + + sort.Slice(nodes, func(i, j int) bool { + return bytes.Compare(nodes[i].key, nodes[j].key) < 0 + }) + + for _, n := range nodes { + fn(n.hash, n) + } +} + func (ndb *nodeDB) String() string { var str string index := 0 diff --git a/pruning_test.go b/pruning_test.go index cd3c9e9a5..711deec50 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -127,30 +127,103 @@ func TestDeleteOrphans(t *testing.T) { // check orphans in recentDB are expected traverseOrphansFromDB(mt.ndb.recentDB, recentFn) - // delete snapshot orphans except latest version + // delete snapshot versions except latest version for j := keepEvery; j < mt.Version(); j += keepEvery { err := mt.DeleteVersion(j) require.Nil(t, err, fmt.Sprintf("Could not delete version %d", j)) } + size := 0 lastfn := func(key, v []byte) { - var fromVersion, toVersion int64 - - // See comment on `orphanKeyFmt`. Note that here, `version` and - // `toVersion` are always equal. - orphanKeyFormat.Scan(key, &toVersion, &fromVersion) - - // only orphans that should exist have toVersion = latestVersion - require.Equal(t, mt.Version(), toVersion, fmt.Sprintf("Unexpected Orphan with toVersion: %d", toVersion)) + size++ } traverseOrphansFromDB(mt.ndb.snapshotDB, lastfn) + require.Equal(t, 0, size, "Orphans still exist in SnapshotDB") + size = 0 // delete all recent orphans escept latest version for k := mt.Version()-keepRecent+1; k < mt.Version(); k++ { err := mt.DeleteVersion(k) require.Nil(t, err, fmt.Sprintf("Could not delete version %d", k)) } traverseOrphansFromDB(mt.ndb.recentDB, lastfn) + require.Equal(t, 0, size, "Orphans still exist in recentDB") + + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), fmt.Sprintf("More nodes in recentDB than expected. KeepEvery: %d, KeepRecent: %d.", keepEvery, keepRecent)) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), fmt.Sprintf("More nodes in snapshotDB than expected. KeepEvery: %d, KeepRecent: %d.", keepEvery, keepRecent)) +} + +func TestDBState(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + keepRecent := int64(5) + keepEvery := int64(1) + mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + + // create 5 versions + for i := 0; i < 5; i++ { + // set 5 keys per version + for j := 0; j < 5; j++ { + key := make([]byte, 8) + val := make([]byte, 8) + binary.BigEndian.PutUint64(key, uint64(rand.Int63())) + binary.BigEndian.PutUint64(val, uint64(rand.Int63())) + mt.Set(key, val) + } + _, _, err := mt.SaveVersion() + require.Nil(t, err, "SaveVersion failed") + } + + require.Equal(t, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) + + for i := 1; i < 5; i++ { + err := mt.DeleteVersion(int64(i)) + require.Nil(t, err, fmt.Sprintf("Could not delete version: %d", i)) + } + + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) +} + +func TestSanity(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + keepRecent := int64(5) + keepEvery := int64(5) + mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + + // create 5 versions + for i := 0; i < 10; i++ { + // set keys per version + mt.Set([]byte(fmt.Sprintf("Key%d", i)), []byte(fmt.Sprintf("Val%d", i))) + _, _, err := mt.SaveVersion() + require.Nil(t, err, "SaveVersion failed") + } + + //require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "SnapshotDB did not save correctly") + + for i := 9; i > 0; i-- { + mt.ndb.DeleteVersionFromRecent(int64(i), true) + mt.ndb.Commit() + } + + size := 0 + fn := func(k, v []byte) { + size++; + } + traverseOrphansFromDB(mt.ndb.recentDB, fn) + require.Equal(t, 0, size, "Not all orphans deleted") + + //require.Equal(t, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "DB sizes should be the same") + + for i := 9; i > 0; i-- { + mt.DeleteVersion(int64(i)) + } + + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) } func TestNoSnapshots(t *testing.T) { From dacbd97426bdb9e06a2f892458549c4036c31ce0 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 22 Jul 2019 16:55:58 -0700 Subject: [PATCH 25/57] fix bugs --- logger.go | 14 ++++- mutable_tree.go | 1 + mutable_tree_test.go | 12 ++++ nodedb.go | 29 +++++----- pruning_test.go | 131 +++++++++++++++++++++++++++++++++++-------- tree_test.go | 9 ++- 6 files changed, 156 insertions(+), 40 deletions(-) diff --git a/logger.go b/logger.go index b53ce34ed..0798fa058 100644 --- a/logger.go +++ b/logger.go @@ -4,8 +4,20 @@ import ( "fmt" ) +var ( + debugging = false +) + func debug(format string, args ...interface{}) { - if false { + if debugging { fmt.Printf(format, args...) } } + +func startDebugger() { + debugging = true +} + +func endDebugger() { + debugging = false +} diff --git a/mutable_tree.go b/mutable_tree.go index 100a0b1ab..e515b27f1 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -420,6 +420,7 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveRoot(tree.root, version) } + tree.ndb.Commit() // Prune nodeDB and delete any pruned versions from tree.versions prunedVersions := tree.ndb.PruneRecentVersions() diff --git a/mutable_tree_test.go b/mutable_tree_test.go index 9a1812be0..9aa1bb314 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -1,6 +1,7 @@ package iavl import ( + "fmt" "bytes" "testing" @@ -30,3 +31,14 @@ func TestDelete(t *testing.T) { k1Value, _, err = tree.GetVersionedWithProof([]byte("k1"), version) require.Equal(t, 0, bytes.Compare([]byte("Fred"), k1Value)) } + +func TestTraverse(t *testing.T) { + memDb := db.NewMemDB() + tree := NewMutableTree(memDb, 0) + + for i := 0; i < 6; i++ { + tree.set([]byte(fmt.Sprintf("k%d", i)), []byte(fmt.Sprintf("v%d", i))) + } + + require.Equal(t, 11, tree.nodeSize(), "Size of tree unexpected") +} diff --git a/nodedb.go b/nodedb.go index 70bf46304..eaf7082d2 100644 --- a/nodedb.go +++ b/nodedb.go @@ -134,13 +134,14 @@ func (ndb *nodeDB) SaveNode(node *Node, flushToDisk bool) { panic(err) } + if !node.saved { + node.saved = true + ndb.recentBatch.Set(ndb.nodeKey(node.hash), buf.Bytes()) + } if flushToDisk { ndb.snapshotBatch.Set(ndb.nodeKey(node.hash), buf.Bytes()) node.persisted = true } - - node.saved = true - ndb.recentBatch.Set(ndb.nodeKey(node.hash), buf.Bytes()) } // Has checks if a hash exists in the database. @@ -180,20 +181,22 @@ func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { return node.hash } - if node.leftNode != nil { + if node.size != 1 { + if node.leftNode == nil { + node.leftNode = ndb.GetNode(node.leftHash) + } + if node.rightNode == nil { + node.rightNode = ndb.GetNode(node.rightHash) + } node.leftHash = ndb.SaveBranch(node.leftNode, flushToDisk) - } - if node.rightNode != nil { node.rightHash = ndb.SaveBranch(node.rightNode, flushToDisk) } node._hash() ndb.SaveNode(node, flushToDisk) - if node.persisted { - node.leftNode = nil - node.rightNode = nil - } + node.leftNode = nil + node.rightNode = nil return node.hash } @@ -227,8 +230,8 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { for hash, fromVersion := range orphans { flushToDisk := false if ndb.keepEvery != 0 { - // if snapshot version in between fromVersion and toVersion, then flush to disk. Or fromVersion == toVersion == snapshotVersion - flushToDisk = fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery || (ndb.isSnapshotVersion(toVersion) && ndb.isSnapshotVersion(fromVersion)) + // if snapshot version in between fromVersion and toVersion INCLUSIVE, then flush to disk. + flushToDisk = fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery || ndb.isSnapshotVersion(fromVersion) } debug("SAVEORPHAN %v-%v %X flushToDisk: %t\n", fromVersion, toVersion, hash, flushToDisk) ndb.saveOrphan([]byte(hash), fromVersion, toVersion, flushToDisk) @@ -259,13 +262,11 @@ func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { ndb.deleteOrphansMem(version) } if ndb.isSnapshotVersion(version) && !memOnly { - debug("DELETE ORPHAN FROM SNAPSHOT\n") predecessor := getPreviousVersionFromDB(version, ndb.snapshotDB) traverseOrphansVersionFromDB(ndb.snapshotDB, version, func(key, hash []byte) { ndb.snapshotDB.Delete(key) ndb.deleteOrphansHelper(ndb.snapshotDB, ndb.snapshotBatch, true, predecessor, key, hash) }) - debug("END DELETE ORPHAN FROM SNAPSHOT\n") } } diff --git a/pruning_test.go b/pruning_test.go index 711deec50..cddaf9b30 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -3,8 +3,8 @@ package iavl import ( "encoding/binary" "fmt" - "os" "math/rand" + "os" "testing" "github.com/stretchr/testify/require" @@ -62,16 +62,16 @@ func TestSave(t *testing.T) { for j := keepEvery; j <= mt.Version(); j += keepEvery { require.True(t, mt.VersionExists(int64(j)), fmt.Sprintf("Expected snapshot version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", j, keepEvery, keepRecent)) } - for k := mt.Version()-keepRecent+1; k <= mt.Version(); k++ { + for k := mt.Version() - keepRecent + 1; k <= mt.Version(); k++ { require.True(t, mt.VersionExists(int64(k)), fmt.Sprintf("Expected recent version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", k, keepEvery, keepRecent)) } // check that there only exists correct number of roots in nodeDB roots, err := mt.ndb.getRoots() require.Nil(t, err, "Error in getRoots") - numRoots := 1000 / keepEvery + keepRecent + numRoots := 1000/keepEvery + keepRecent // decrement if there is overlap between snapshot and recent versions - if 1000 % keepEvery == 0 { + if 1000%keepEvery == 0 { numRoots-- } require.Equal(t, numRoots, int64(len(roots)), "nodeDB does not contain expected number of roots") @@ -83,10 +83,10 @@ func TestDeleteOrphans(t *testing.T) { keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (rand.Int63n(3) + 1) * 100 - mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + mt := NewMutableTreePruningOpts(db, mdb, 500, keepEvery, keepRecent) - // create 1000 versions - for i := 0; i < 1000; i++ { + // create 1200 versions (multiple of any possible snapshotting version) + for i := 0; i < 1200; i++ { // set 5 keys per version for j := 0; j < 5; j++ { key := make([]byte, 8) @@ -121,7 +121,7 @@ func TestDeleteOrphans(t *testing.T) { orphanKeyFormat.Scan(key, &toVersion, &fromVersion) // toVersion must exist in recentDB - require.True(t, toVersion > mt.Version() - keepRecent, fmt.Sprintf("Orphan in recentDB has unexpected fromVersion: %d. Should have been deleted", fromVersion)) + require.True(t, toVersion > mt.Version()-keepRecent, fmt.Sprintf("Orphan in recentDB has unexpected fromVersion: %d. Should have been deleted", fromVersion)) } // check orphans in recentDB are expected @@ -142,7 +142,7 @@ func TestDeleteOrphans(t *testing.T) { size = 0 // delete all recent orphans escept latest version - for k := mt.Version()-keepRecent+1; k < mt.Version(); k++ { + for k := mt.Version() - keepRecent + 1; k < mt.Version(); k++ { err := mt.DeleteVersion(k) require.Nil(t, err, fmt.Sprintf("Could not delete version %d", k)) } @@ -186,43 +186,130 @@ func TestDBState(t *testing.T) { require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) } -func TestSanity(t *testing.T) { +func TestSanity1(t *testing.T) { db, mdb, close := getTestDBs() defer close() - keepRecent := int64(5) + keepRecent := int64(1) keepEvery := int64(5) mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) // create 5 versions - for i := 0; i < 10; i++ { + for i := 0; i < 5; i++ { // set keys per version - mt.Set([]byte(fmt.Sprintf("Key%d", i)), []byte(fmt.Sprintf("Val%d", i))) + // set 2 keys per version + for j := 0; j < 2; j++ { + key := []byte(fmt.Sprintf("%d: Key:v%d:i%d", rand.Int63(), i+1, j)) + val := []byte(fmt.Sprintf("Val:v%d:i%d", i, j)) + mt.Set(key, val) + } + _, _, err := mt.SaveVersion() require.Nil(t, err, "SaveVersion failed") } - //require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "SnapshotDB did not save correctly") + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "SnapshotDB did not save correctly") - for i := 9; i > 0; i-- { + for i := 1; i < 5; i++ { mt.ndb.DeleteVersionFromRecent(int64(i), true) mt.ndb.Commit() } + require.Equal(t, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "DB sizes should be the same") + size := 0 fn := func(k, v []byte) { - size++; + size++ } traverseOrphansFromDB(mt.ndb.recentDB, fn) require.Equal(t, 0, size, "Not all orphans deleted") - //require.Equal(t, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "DB sizes should be the same") + size = 0 + traverseOrphansFromDB(mt.ndb.snapshotDB, fn) + require.Equal(t, 0, size, "Not all orphans in snapshotDBdeleted") + + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) +} + +func TestSanity2(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() - for i := 9; i > 0; i-- { + keepRecent := int64(1) + keepEvery := int64(5) + mt := NewMutableTreePruningOpts(db, mdb, 0, keepEvery, keepRecent) + + // create 5 versions + for i := 0; i < 5; i++ { + // set keys per version + for j := 0; j < 2; j++ { + mt.Set([]byte(fmt.Sprintf("%dKey%d|%d", rand.Int63(), i, j)), []byte(fmt.Sprintf("Val%d%d", i, j))) + } + _, _, err := mt.SaveVersion() + require.Nil(t, err, "SaveVersion failed") + } + + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "SnapshotDB did not save correctly") + + size := 0 + fn := func(k, v []byte) { + size++ + } + traverseOrphansFromDB(mt.ndb.snapshotDB, fn) + require.Equal(t, 0, size, "Not all orphans deleted") + + size = 0 + traverseOrphansFromDB(mt.ndb.recentDB, fn) + require.Equal(t, 0, size, "Not all orphans deleted from RecentDB") + + require.Equal(t, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "DB sizes should be the same") + + for i := 1; i < 5; i++ { mt.DeleteVersion(int64(i)) } - require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) + require.Equal(t, mt.nodeSize()+size, len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) +} + +func TestSanity3(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + keepRecent := int64(4) + keepEvery := int64(100) + mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + + // create 1000 versions + numSnapNodes := 0 + for i := 0; i < 200; i++ { + // set 5 keys per version + var key, val []byte + for j := 0; j < 5; j++ { + key = []byte(fmt.Sprintf("%d: Key:v%d:i%d", rand.Int63(), i+1, j)) + val = []byte(fmt.Sprintf("Val:v%d:i%d", i, j)) + } + mt.Set(key, val) + _, _, err := mt.SaveVersion() + if int64(i+1)%keepEvery == 0 { + numSnapNodes += mt.nodeSize() + } + require.Nil(t, err, "SaveVersion failed") + } + + fn := func(n *Node) bool { + if n.version <= 100 { + numSnapNodes-- + } + return false + } + mt.root.traverse(mt.ImmutableTree, true, fn) + + require.Equal(t, numSnapNodes, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) + + mt.DeleteVersion(100) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) } @@ -230,7 +317,7 @@ func TestNoSnapshots(t *testing.T) { db, mdb, close := getTestDBs() defer close() - keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB + keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB mt := NewMutableTreePruningOpts(db, mdb, 5, 0, keepRecent) // test no snapshots for i := 0; i < 50; i++ { @@ -251,11 +338,11 @@ func TestNoSnapshots(t *testing.T) { for i := 0; int64(i) < keepRecent; i++ { seen := false for _, v := range versions { - if v == int(mt.Version()) - i { + if v == int(mt.Version())-i { seen = true } } - require.True(t, seen, fmt.Sprintf("Version %d is not available even though it is recent", mt.Version() - int64(i))) + require.True(t, seen, fmt.Sprintf("Version %d is not available even though it is recent", mt.Version()-int64(i))) } size := 0 diff --git a/tree_test.go b/tree_test.go index 059addbe7..e5728bd49 100644 --- a/tree_test.go +++ b/tree_test.go @@ -542,7 +542,10 @@ func TestVersionedTreeOrphanDeleting(t *testing.T) { func TestVersionedTreeSpecialCase(t *testing.T) { require := require.New(t) - tree := getTestTree(100) + d, closeDB := getTestDB() + defer closeDB() + + tree := NewMutableTree(d, 0) tree.Set([]byte("key1"), []byte("val0")) tree.Set([]byte("key2"), []byte("val0")) @@ -563,7 +566,7 @@ func TestVersionedTreeSpecialCase(t *testing.T) { func TestVersionedTreeSpecialCase2(t *testing.T) { require := require.New(t) - + d := db.NewMemDB() tree := NewMutableTree(d, 100) @@ -820,7 +823,7 @@ func TestVersionedCheckpointsSpecialCase3(t *testing.T) { func TestVersionedCheckpointsSpecialCase4(t *testing.T) { tree := NewMutableTree(db.NewMemDB(), 0) - + tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) tree.Set([]byte("H"), []byte("7a9En4uw")) From 67f9e19f79ca4bd9b71cdfd51e9c172a8ce12c26 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Tue, 23 Jul 2019 09:55:51 -0700 Subject: [PATCH 26/57] address initial comments --- CHANGELOG.md | 4 ---- CHANGELOG_PENDING.md | 4 ++++ PRUNING.md | 4 +++- immutable_tree.go | 4 +++- mutable_tree.go | 3 ++- nodedb.go | 1 + testutils_test.go | 2 +- 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54bdb30dd..2237afbbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,5 @@ # Changelog -## Pending - -- [IAVL] Ability to store temporary versions in memory only, and flush to disk every X versions. This should greatly reduce IO requirements and disk storage. (@mattkanwisher Loom Network) - ## 0.12.3 (July 12, 2019) Special thanks to external contributors on this release: diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 24d60b415..f5cb99c34 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -6,6 +6,10 @@ Special thanks to external contributors on this release: ### BREAKING CHANGES +- [/#158] NodeDB constructor must provide `keepRecent` and `keepEvery` fields to define PruningStrategy. All Save functionality must specify whether they should flushToDisk as well using `flushToDisk` boolean argument. All Delete functionality must specify whether object should be deleted from memory only using the `memOnly` boolean argument. + ### IMPROVEMENTS - [/#46](https://github.com/tendermint/iavl/issues/46) Removal of all instances of cmn from tendermint in Ival repo + +- [/#158] Add constructor to IAVL that can define a pruning strategy. Only persist to disk snapshot versions and keep a specified number of versions in memDB. This greatly reduces IO load of IAVL. diff --git a/PRUNING.md b/PRUNING.md index 6c8013cd5..9261aa498 100644 --- a/PRUNING.md +++ b/PRUNING.md @@ -1,3 +1,5 @@ +# Pruning + Setting Pruning fields in the IAVL tree can optimize performance by only writing versions to disk if they are meant to be persisted indefinitely. Versions that are known to be deleted eventually are temporarily held in memory until they are ready to be pruned. This greatly reduces the I/O load of IAVL. We can set custom pruning fields in IAVL using: `NewMutableTreePruningOpts` @@ -16,7 +18,7 @@ keepEvery int64n // Saves version to disk periodically keepRecent int64 // Saves recent versions in memory ``` -If version is not going to be persisted to disk, the version is simply saved in `recentDB` (typically a memDB) +If version is not going to be persisted to disk, the version is simply saved in `recentDB` (typically a `memDB`) If version is persisted to disk, the version is written to `recentDB` **and** `snapshotDB` (typically `levelDB`) #### Orphans: diff --git a/immutable_tree.go b/immutable_tree.go index ca70505ee..d94ee390c 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -16,7 +16,7 @@ type ImmutableTree struct { version int64 } -// NewImmutableTree creates both in-memory and persistent instances +// NewImmutableTree creates both in-memory and persistent instances. Default behavior snapshots every version func NewImmutableTree(db dbm.DB, cacheSize int) *ImmutableTree { if db == nil { // In-memory Tree. @@ -29,6 +29,8 @@ func NewImmutableTree(db dbm.DB, cacheSize int) *ImmutableTree { } } +// NewImmutableTreePruningOpts creates ImmutableTree with specified pruning strategy. +// Persists every `keepEvery` version to snapDB and saves last `keepRecent` versions to recentDB func NewImmutableTreePruningOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *ImmutableTree { return &ImmutableTree{ // NodeDB-backed Tree. diff --git a/mutable_tree.go b/mutable_tree.go index 8f5ffbec3..e20ede930 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -22,7 +22,7 @@ type MutableTree struct { ndb *nodeDB } -// NewMutableTree returns a new tree with the specified cache size and datastore and pruning options +// NewMutableTree returns a new tree with the specified cache size and datastore // To maintain backwards compatibility, this function will initialize PruningStrategy{keepEvery: 1, keepRecent: 0} func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { // memDB is initialized but should never be written to @@ -30,6 +30,7 @@ func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { return NewMutableTreePruningOpts(db, memDB, cacheSize, 1, 0) } +// NewMutableTreePruningOpts returns a new tree with the specified cache size, datastores and pruning options func NewMutableTreePruningOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *MutableTree { ndb := newNodeDB(snapDB, recentDB, cacheSize, keepEvery, keepRecent) head := &ImmutableTree{ndb: ndb} diff --git a/nodedb.go b/nodedb.go index f8a9d9513..f3efa0cc9 100644 --- a/nodedb.go +++ b/nodedb.go @@ -141,6 +141,7 @@ func (ndb *nodeDB) SaveNode(node *Node, flushToDisk bool) { if flushToDisk { ndb.snapshotBatch.Set(ndb.nodeKey(node.hash), buf.Bytes()) node.persisted = true + node.saved = true } } diff --git a/testutils_test.go b/testutils_test.go index 3d0bb3f56..976f98f9b 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -30,7 +30,7 @@ func b2i(bz []byte) int { // Construct a MutableTree with random pruning parameters func getTestTree(cacheSize int) *MutableTree { - keepRecent := mrand.Int63n(8) + 2 //keep at least 2 versions in memDB + keepRecent := mrand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (mrand.Int63n(3) + 1) * 100 // snapshot every {100,200,300} versions // Use MemDB for recentDB and snapshotDB From b5d09a737f83113296efba532ba9ff878d664918 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Wed, 24 Jul 2019 15:11:55 -0700 Subject: [PATCH 27/57] add replace/remove pruning tests. improve remove efficiency --- mutable_tree.go | 11 +++--- pruning_test.go | 96 ++++++++++++++++++++++++++++++++++++++++++------- tree_test.go | 6 ++-- 3 files changed, 93 insertions(+), 20 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index e20ede930..742be39a3 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -208,10 +208,12 @@ func (tree *MutableTree) recursiveRemove(node *Node, key []byte) ([]byte, *Node, if len(orphaned) == 0 { return node.hash, node, nil, value, orphaned - } else if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed - return node.rightHash, node.rightNode, node.key, value, orphaned } orphaned = append(orphaned, node) + if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed + orphaned = append(orphaned, node) + return node.rightHash, node.rightNode, node.key, value, orphaned + } newNode := node.clone(version) newNode.leftHash, newNode.leftNode = newLeftHash, newLeftNode @@ -225,10 +227,11 @@ func (tree *MutableTree) recursiveRemove(node *Node, key []byte) ([]byte, *Node, if len(orphaned) == 0 { return node.hash, node, nil, value, orphaned - } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed - return node.leftHash, node.leftNode, nil, value, orphaned } orphaned = append(orphaned, node) + if newRightHash == nil && newRightNode == nil { // right node held value, was removed + return node.leftHash, node.leftNode, nil, value, orphaned + } newNode := node.clone(version) newNode.rightHash, newNode.rightNode = newRightHash, newRightNode diff --git a/pruning_test.go b/pruning_test.go index 93624e2c7..c1092a284 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -50,7 +50,7 @@ func TestSave(t *testing.T) { ver := int64(v) // check that version is supposed to exist given pruning strategy require.True(t, ver%keepEvery == 0 || mt.Version()-ver <= keepRecent, - fmt.Sprintf("Version: %d should not exist. KeepEvery: %d, KeepRecent: %d", v, keepEvery, keepRecent)) + "Version: %d should not exist. KeepEvery: %d, KeepRecent: %d", v, keepEvery, keepRecent) // check that root exists in nodeDB lv, err := mt.LazyLoadVersion(ver) @@ -60,10 +60,10 @@ func TestSave(t *testing.T) { // check all expected versions are available. for j := keepEvery; j <= mt.Version(); j += keepEvery { - require.True(t, mt.VersionExists(int64(j)), fmt.Sprintf("Expected snapshot version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", j, keepEvery, keepRecent)) + require.True(t, mt.VersionExists(int64(j)), "Expected snapshot version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", j, keepEvery, keepRecent) } for k := mt.Version() - keepRecent + 1; k <= mt.Version(); k++ { - require.True(t, mt.VersionExists(int64(k)), fmt.Sprintf("Expected recent version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", k, keepEvery, keepRecent)) + require.True(t, mt.VersionExists(int64(k)), "Expected recent version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", k, keepEvery, keepRecent) } // check that there only exists correct number of roots in nodeDB @@ -83,7 +83,7 @@ func TestDeleteOrphans(t *testing.T) { keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (rand.Int63n(3) + 1) * 100 - mt := NewMutableTreePruningOpts(db, mdb, 500, keepEvery, keepRecent) + mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) // create 1200 versions (multiple of any possible snapshotting version) for i := 0; i < 1200; i++ { @@ -107,7 +107,7 @@ func TestDeleteOrphans(t *testing.T) { orphanKeyFormat.Scan(key, &toVersion, &fromVersion) // toVersion must be snapshotVersion - require.True(t, toVersion%keepEvery == 0, fmt.Sprintf("Orphan in snapshotDB has unexpected toVersion: %d. Should never have been persisted", toVersion)) + require.True(t, toVersion%keepEvery == 0, "Orphan in snapshotDB has unexpected toVersion: %d. Should never have been persisted", toVersion) } // check orphans in snapshotDB are expected @@ -121,7 +121,7 @@ func TestDeleteOrphans(t *testing.T) { orphanKeyFormat.Scan(key, &toVersion, &fromVersion) // toVersion must exist in recentDB - require.True(t, toVersion > mt.Version()-keepRecent, fmt.Sprintf("Orphan in recentDB has unexpected fromVersion: %d. Should have been deleted", fromVersion)) + require.True(t, toVersion > mt.Version()-keepRecent, "Orphan in recentDB has unexpected fromVersion: %d. Should have been deleted", fromVersion) } // check orphans in recentDB are expected @@ -130,7 +130,7 @@ func TestDeleteOrphans(t *testing.T) { // delete snapshot versions except latest version for j := keepEvery; j < mt.Version(); j += keepEvery { err := mt.DeleteVersion(j) - require.Nil(t, err, fmt.Sprintf("Could not delete version %d", j)) + require.Nil(t, err, "Could not delete version %d", j) } size := 0 @@ -144,13 +144,82 @@ func TestDeleteOrphans(t *testing.T) { // delete all recent orphans escept latest version for k := mt.Version() - keepRecent + 1; k < mt.Version(); k++ { err := mt.DeleteVersion(k) - require.Nil(t, err, fmt.Sprintf("Could not delete version %d", k)) + require.Nil(t, err, "Could not delete version %d", k) } traverseOrphansFromDB(mt.ndb.recentDB, lastfn) require.Equal(t, 0, size, "Orphans still exist in recentDB") - require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), fmt.Sprintf("More nodes in recentDB than expected. KeepEvery: %d, KeepRecent: %d.", keepEvery, keepRecent)) - require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), fmt.Sprintf("More nodes in snapshotDB than expected. KeepEvery: %d, KeepRecent: %d.", keepEvery, keepRecent)) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "More nodes in recentDB than expected. KeepEvery: %d, KeepRecent: %d.", keepEvery, keepRecent) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "More nodes in snapshotDB than expected. KeepEvery: %d, KeepRecent: %d.", keepEvery, keepRecent) +} + +func TestReplaceKeys(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + keepRecent := int64(1) //keep 1 version in memDB + keepEvery := int64(5) + mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + + // Replace the same 10 keys with different values + for i := 0; i < 10; i++ { + for j := 0; j < 10; j++ { + mt.Set([]byte(fmt.Sprintf("%d", j)), []byte(fmt.Sprintf("Val:%d::Version:%d", j, i+1))) + } + _, _, err := mt.SaveVersion() + require.Nil(t, err, "Could not save version %d", i) + } + + // check that existing versions have expected values + for i := 0; i < 10; i++ { + _, val := mt.GetVersioned([]byte(fmt.Sprintf("%d", i)), 5) + require.Equal(t, fmt.Sprintf("Val:%d::Version:5", i), string(val), "Value from Version 5 unexpected") + _, val = mt.GetVersioned([]byte(fmt.Sprintf("%d", i)), 10) + require.Equal(t, fmt.Sprintf("Val:%d::Version:10", i), string(val), "Value from Version 10 unexpected") + } + + // check that all pruned versions have nil values + for v := 1; v < 10; v++ { + if v != 5 { + for i := 0; i < 10; i++ { + _, val := mt.GetVersioned([]byte(fmt.Sprintf("%d", i)), int64(v)) + require.Nil(t, val, "Pruned version: %d still has non-nil value: %v in db", v, val) + } + } + } +} + +func TestRemoveKeys(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + keepRecent := int64(1) //keep 1 version in memDB + keepEvery := int64(10) + mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + + for v := 0; v < 10; v++ { + for i := 0; i < 10; i++ { + mt.Set([]byte(fmt.Sprintf("v%d:%d", v, i)), []byte(fmt.Sprintf("Val:v:%d:%d", v, i))) + } + mt.SaveVersion() + } + + numNodes := mt.nodeSize() + + for v := 0; v < 10; v++ { + for i := 0; i < 10; i++ { + key := fmt.Sprintf("v%d:%d", v, i) + _, removed := mt.Remove([]byte(key)) + require.True(t, removed, "Key %s could not be removed", key) + } + mt.SaveVersion() + } + + require.Equal(t, numNodes, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "Number of Nodes in snapshotDB are unexpected") + + // Delete only non-empty tree in snapshotDB + mt.DeleteVersion(10) + require.Equal(t, 0, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "Still have nodes in snapshotDB") } func TestDBState(t *testing.T) { @@ -179,7 +248,7 @@ func TestDBState(t *testing.T) { for i := 1; i < 5; i++ { err := mt.DeleteVersion(int64(i)) - require.Nil(t, err, fmt.Sprintf("Could not delete version: %d", i)) + require.Nil(t, err, "Could not delete version: %d", i) } require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) @@ -313,6 +382,7 @@ func TestSanity3(t *testing.T) { require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) } +/* Test Pruning Edge Cases */ func TestNoSnapshots(t *testing.T) { db, mdb, close := getTestDBs() defer close() @@ -342,7 +412,7 @@ func TestNoSnapshots(t *testing.T) { seen = true } } - require.True(t, seen, fmt.Sprintf("Version %d is not available even though it is recent", mt.Version()-int64(i))) + require.True(t, seen, "Version %d is not available even though it is recent", mt.Version()-int64(i)) } size := 0 @@ -388,6 +458,6 @@ func TestNoRecents(t *testing.T) { seen = true } } - require.True(t, seen, fmt.Sprintf("Version %d is not available even though it is snpashhot version", i)) + require.True(t, seen, "Version %d is not available even though it is snpashhot version", i) } } diff --git a/tree_test.go b/tree_test.go index 58afbf047..6f4c5c8b5 100644 --- a/tree_test.go +++ b/tree_test.go @@ -339,8 +339,8 @@ func TestVersionedTree(t *testing.T) { require.Len(nodes2, 5, "db should have grown in size") require.Len(tree.ndb.orphans(), 3, "db should have three orphans") - // Create two more orphans. - tree.Remove([]byte("key1")) + // Create three more orphans. + tree.Remove([]byte("key1")) // orphans both leaf node and inner node containing "key1" and "key2" tree.Set([]byte("key2"), []byte("val2")) hash3, v3, _ := tree.SaveVersion() @@ -359,7 +359,7 @@ func TestVersionedTree(t *testing.T) { nodes3 := tree.ndb.leafNodes() require.Len(nodes3, 6, "wrong number of nodes") - require.Len(tree.ndb.orphans(), 6, "wrong number of orphans") + require.Len(tree.ndb.orphans(), 7, "wrong number of orphans") hash4, _, _ := tree.SaveVersion() require.EqualValues(hash3, hash4) From f4b9941f373bb8ac81bc3a2de5f050d29df4ebf2 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Thu, 25 Jul 2019 14:21:35 -0700 Subject: [PATCH 28/57] update bench_test --- benchmarks/bench_test.go | 64 +++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index f9ff8a812..89ef7d011 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -20,8 +20,8 @@ func randBytes(length int) []byte { return key } -func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { - t := iavl.NewMutableTree(db, size) +func prepareTree(b *testing.B, snapdb db.DB, memdb db.DB, keepEvery int64, keepRecent int64, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { + t := iavl.NewMutableTreePruningOpts(snapdb, memdb, size, keepEvery, keepRecent) keys := make([][]byte, size) for i := 0; i < size; i++ { @@ -157,13 +157,16 @@ type benchmark struct { keyLen, dataLen int } +type pruningstrat struct { + keepEvery, keepRecent int64 +} + func BenchmarkMedium(b *testing.B) { benchmarks := []benchmark{ {"memdb", 100000, 100, 16, 40}, {"goleveldb", 100000, 100, 16, 40}, // FIXME: this crashes on init! Either remove support, or make it work. // {"cleveldb", 100000, 100, 16, 40}, - {"leveldb", 100000, 100, 16, 40}, } runBenchmarks(b, benchmarks) } @@ -174,7 +177,6 @@ func BenchmarkSmall(b *testing.B) { {"goleveldb", 1000, 100, 4, 10}, // FIXME: this crashes on init! Either remove support, or make it work. // {"cleveldb", 100000, 100, 16, 40}, - {"leveldb", 1000, 100, 4, 10}, } runBenchmarks(b, benchmarks) } @@ -185,7 +187,6 @@ func BenchmarkLarge(b *testing.B) { {"goleveldb", 1000000, 100, 16, 40}, // FIXME: this crashes on init! Either remove support, or make it work. // {"cleveldb", 100000, 100, 16, 40}, - {"leveldb", 1000000, 100, 16, 40}, } runBenchmarks(b, benchmarks) } @@ -214,28 +215,37 @@ func BenchmarkLevelDBLargeData(b *testing.B) { } func runBenchmarks(b *testing.B, benchmarks []benchmark) { - for _, bb := range benchmarks { - prefix := fmt.Sprintf("%s-%d-%d-%d-%d", bb.dbType, bb.initSize, - bb.blockSize, bb.keyLen, bb.dataLen) - - // prepare a dir for the db and cleanup afterwards - dirName := fmt.Sprintf("./%s-db", prefix) - defer func() { - err := os.RemoveAll(dirName) - if err != nil { - b.Errorf("%+v\n", err) + pruningStrategies := []pruningstrat{ + {1, 0}, // default pruning strategy + {0, 1}, // keep single recent version + {100, 5}, // simple pruning + {1000, 10}, // average pruning + {1000, 1}, // extreme pruning + } + for _, ps := range pruningStrategies { + for _, bb := range benchmarks { + prefix := fmt.Sprintf("%s-%d-%d-%d-%d-%d-%d", bb.dbType, ps.keepEvery, ps.keepRecent, + bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen) + + // prepare a dir for the db and cleanup afterwards + dirName := fmt.Sprintf("./%s-db", prefix) + defer func() { + err := os.RemoveAll(dirName) + if err != nil { + b.Errorf("%+v\n", err) + } + }() + + // note that "" leads to nil backing db! + var d db.DB + if bb.dbType != "nodb" { + d = db.NewDB("test", bb.dbType, dirName) + defer d.Close() } - }() - - // note that "" leads to nil backing db! - var d db.DB - if bb.dbType != "nodb" { - d = db.NewDB("test", bb.dbType, dirName) - defer d.Close() + b.Run(prefix, func(sub *testing.B) { + runSuite(sub, d, ps.keepEvery, ps.keepRecent, bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen) + }) } - b.Run(prefix, func(sub *testing.B) { - runSuite(sub, d, bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen) - }) } } @@ -248,12 +258,12 @@ func memUseMB() float64 { return mb } -func runSuite(b *testing.B, d db.DB, initSize, blockSize, keyLen, dataLen int) { +func runSuite(b *testing.B, d db.DB, keepEvery int64, keepRecent int64, initSize, blockSize, keyLen, dataLen int) { // measure mem usage runtime.GC() init := memUseMB() - t, keys := prepareTree(b, d, initSize, keyLen, dataLen) + t, keys := prepareTree(b, d, db.NewMemDB(), keepEvery, keepRecent, initSize, keyLen, dataLen) used := memUseMB() - init fmt.Printf("Init Tree took %0.2f MB\n", used) From 0a1f25e5636159930933ca8b0358db66699617da Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Thu, 25 Jul 2019 15:03:18 -0700 Subject: [PATCH 29/57] add some more pruning options --- benchmarks/bench_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 89ef7d011..c4c23f686 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -216,11 +216,13 @@ func BenchmarkLevelDBLargeData(b *testing.B) { func runBenchmarks(b *testing.B, benchmarks []benchmark) { pruningStrategies := []pruningstrat{ - {1, 0}, // default pruning strategy - {0, 1}, // keep single recent version - {100, 5}, // simple pruning - {1000, 10}, // average pruning - {1000, 1}, // extreme pruning + {0, 0} // prune everything + {1, 0}, // default pruning strategy + {0, 1}, // keep single recent version + {100, 5}, // simple pruning + {1000, 10}, // average pruning + {1000, 1}, // extreme pruning + {10000, 100} // SDK pruning } for _, ps := range pruningStrategies { for _, bb := range benchmarks { From 097f2c5046706884425549f5e7562edcdddacec4 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Thu, 25 Jul 2019 15:06:36 -0700 Subject: [PATCH 30/57] add commas --- benchmarks/bench_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index c4c23f686..c33984800 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -216,13 +216,13 @@ func BenchmarkLevelDBLargeData(b *testing.B) { func runBenchmarks(b *testing.B, benchmarks []benchmark) { pruningStrategies := []pruningstrat{ - {0, 0} // prune everything - {1, 0}, // default pruning strategy - {0, 1}, // keep single recent version - {100, 5}, // simple pruning - {1000, 10}, // average pruning - {1000, 1}, // extreme pruning - {10000, 100} // SDK pruning + {0, 0}, // prune everything + {1, 0}, // default pruning strategy + {0, 1}, // keep single recent version + {100, 5}, // simple pruning + {1000, 10}, // average pruning + {1000, 1}, // extreme pruning + {10000, 100}, // SDK pruning } for _, ps := range pruningStrategies { for _, bb := range benchmarks { From 7bf79c08aa3de0f82be18b72b70029efd731c211 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 26 Jul 2019 11:32:20 -0700 Subject: [PATCH 31/57] add pruning benchmark tests --- Makefile | 3 ++ benchmarks/bench_test.go | 5 --- benchmarks/prune_test.go | 82 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 benchmarks/prune_test.go diff --git a/Makefile b/Makefile index 06de1370a..97a30f0ac 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,9 @@ fullbench: go test -bench=Mem . && \ go test -timeout=60m -bench=LevelDB . +benchprune: + cd benchmarks && \ + go test -bench=PruningStrategies # note that this just profiles the in-memory version, not persistence profile: diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index c33984800..df16ec3ee 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -157,10 +157,6 @@ type benchmark struct { keyLen, dataLen int } -type pruningstrat struct { - keepEvery, keepRecent int64 -} - func BenchmarkMedium(b *testing.B) { benchmarks := []benchmark{ {"memdb", 100000, 100, 16, 40}, @@ -216,7 +212,6 @@ func BenchmarkLevelDBLargeData(b *testing.B) { func runBenchmarks(b *testing.B, benchmarks []benchmark) { pruningStrategies := []pruningstrat{ - {0, 0}, // prune everything {1, 0}, // default pruning strategy {0, 1}, // keep single recent version {100, 5}, // simple pruning diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go new file mode 100644 index 000000000..09d88f6be --- /dev/null +++ b/benchmarks/prune_test.go @@ -0,0 +1,82 @@ +package benchmarks + +import ( + "fmt" + "os" + "runtime" + "testing" + + db "github.com/tendermint/tm-cmn/db" +) + +type pruningstrat struct { + keepEvery, keepRecent int64 +} + +// To test effect of pruning strategy, we must measure time to execute many blocks +// Execute 50000 blocks with the given IAVL tree's pruning strategy +func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int64, keyLen, dataLen int) { + // prepare a dir for the db and cleanup afterwards + dirName := fmt.Sprintf("./%s-db", prefix) + defer func() { + err := os.RemoveAll(dirName) + if err != nil { + b.Errorf("%+v\n", err) + } + }() + + runtime.GC() + + // always initialize tree with goleveldb as snapshotDB and memDB as recentDB + snapDB := db.NewDB("test", "goleveldb", dirName) + defer snapDB.Close() + + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + memSize := mem.Alloc + maxVersion := 0 + + // reset timer after initialization logic + b.ResetTimer() + t, _ := prepareTree(b, snapDB, db.NewMemDB(), keepEvery, keepRecent, 5, keyLen, dataLen) + // create 50000 versions + for i := 0; i < 50000; i++ { + // create 5 keys per version + for j := 0; j < 5; j++ { + t.Set(randBytes(keyLen), randBytes(dataLen)) + } + _, _, err := t.SaveVersion() + if err != nil { + b.Errorf("Can't save version %d: %v", i, err) + } + // Pause timer to garbage-collect and remeasure memory usage + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&mem) + // update memSize if it has increased after saveVersion + if memSize < mem.Alloc { + memSize = mem.Alloc + maxVersion = i + } + b.StartTimer() + } + fmt.Printf("Maxmimum Memory usage was %0.2f MB at height %d\n", float64(memSize)/1000000, maxVersion) +} + +func BenchmarkPruningStrategies(b *testing.B) { + ps := []pruningstrat{ + {1, 0}, // default pruning strategy + {0, 1}, // keep single recent version + {100, 5}, // simple pruning + {1000, 10}, // average pruning + {1000, 1}, // extreme pruning + {10000, 100}, // SDK pruning + } + for _, ps := range ps { + prefix := fmt.Sprintf("PruningStrategy{%d-%d}-KeyLen:%d-DataLen:%d", ps.keepEvery, ps.keepRecent, 16, 40) + + b.Run(prefix, func(sub *testing.B) { + runBlockChain(sub, prefix, ps.keepEvery, ps.keepRecent, 16, 40) + }) + } +} From 48e714f821b862e04bdf9afe47dc9ab18c34df86 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 26 Jul 2019 17:07:36 -0700 Subject: [PATCH 32/57] slightly more reasonable pruning benchmarks --- Makefile | 2 +- benchmarks/prune_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 97a30f0ac..122c2e35d 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ fullbench: benchprune: cd benchmarks && \ - go test -bench=PruningStrategies + go test -bench=PruningStrategies -timeout=24h # note that this just profiles the in-memory version, not persistence profile: diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index 09d88f6be..9b08f0856 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -14,7 +14,7 @@ type pruningstrat struct { } // To test effect of pruning strategy, we must measure time to execute many blocks -// Execute 50000 blocks with the given IAVL tree's pruning strategy +// Execute 30000 blocks with the given IAVL tree's pruning strategy func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int64, keyLen, dataLen int) { // prepare a dir for the db and cleanup afterwards dirName := fmt.Sprintf("./%s-db", prefix) @@ -39,8 +39,8 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 // reset timer after initialization logic b.ResetTimer() t, _ := prepareTree(b, snapDB, db.NewMemDB(), keepEvery, keepRecent, 5, keyLen, dataLen) - // create 50000 versions - for i := 0; i < 50000; i++ { + // create 30000 versions + for i := 0; i < 30000; i++ { // create 5 keys per version for j := 0; j < 5; j++ { t.Set(randBytes(keyLen), randBytes(dataLen)) From 67a413101df6eb42047a7b9b93ec3b99762f5349 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 29 Jul 2019 10:46:15 -0700 Subject: [PATCH 33/57] options refactor --- benchmarks/prune_test.go | 49 +++++++++++++++++++++------------------- immutable_tree.go | 9 ++++---- mutable_tree.go | 8 +++---- nodedb.go | 47 ++++++++++++++++++++++---------------- options.go | 36 +++++++++++++++++++++++++++++ pruning_test.go | 34 ++++++++++++++-------------- testutils_test.go | 4 +++- 7 files changed, 119 insertions(+), 68 deletions(-) create mode 100644 options.go diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index 9b08f0856..5fc6cc4cc 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -6,6 +6,7 @@ import ( "runtime" "testing" + "github.com/syndtr/goleveldb/opts" db "github.com/tendermint/tm-cmn/db" ) @@ -28,19 +29,20 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 runtime.GC() // always initialize tree with goleveldb as snapshotDB and memDB as recentDB - snapDB := db.NewDB("test", "goleveldb", dirName) + dbOptions := opts.Options{} + snapDB := NewGoLevelDBWithOpts("test", dirName, &dbOptions) defer snapDB.Close() - var mem runtime.MemStats - runtime.ReadMemStats(&mem) - memSize := mem.Alloc - maxVersion := 0 + // var mem runtime.MemStats + // runtime.ReadMemStats(&mem) + // memSize := mem.Alloc + // maxVersion := 0 // reset timer after initialization logic b.ResetTimer() t, _ := prepareTree(b, snapDB, db.NewMemDB(), keepEvery, keepRecent, 5, keyLen, dataLen) // create 30000 versions - for i := 0; i < 30000; i++ { + for i := 0; i < b.N; i++ { // create 5 keys per version for j := 0; j < 5; j++ { t.Set(randBytes(keyLen), randBytes(dataLen)) @@ -49,28 +51,29 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 if err != nil { b.Errorf("Can't save version %d: %v", i, err) } - // Pause timer to garbage-collect and remeasure memory usage - b.StopTimer() - runtime.GC() - runtime.ReadMemStats(&mem) - // update memSize if it has increased after saveVersion - if memSize < mem.Alloc { - memSize = mem.Alloc - maxVersion = i - } - b.StartTimer() + // // Pause timer to garbage-collect and remeasure memory usage + // b.StopTimer() + // runtime.GC() + // runtime.ReadMemStats(&mem) + // // update memSize if it has increased after saveVersion + // if memSize < mem.Alloc { + // memSize = mem.Alloc + // maxVersion = i + // } + // b.StartTimer() } - fmt.Printf("Maxmimum Memory usage was %0.2f MB at height %d\n", float64(memSize)/1000000, maxVersion) + //fmt.Printf("Maxmimum Memory usage was %0.2f MB at height %d\n", float64(memSize)/1000000, maxVersion) + b.StopTimer() } func BenchmarkPruningStrategies(b *testing.B) { ps := []pruningstrat{ - {1, 0}, // default pruning strategy - {0, 1}, // keep single recent version - {100, 5}, // simple pruning - {1000, 10}, // average pruning - {1000, 1}, // extreme pruning - {10000, 100}, // SDK pruning + {1, 0}, // default pruning strategy + {0, 1}, // keep single recent version + {100, 5}, // simple pruning + // {1000, 10}, // average pruning + // {1000, 1}, // extreme pruning + // {10000, 100}, // SDK pruning } for _, ps := range ps { prefix := fmt.Sprintf("PruningStrategy{%d-%d}-KeyLen:%d-DataLen:%d", ps.keepEvery, ps.keepRecent, 16, 40) diff --git a/immutable_tree.go b/immutable_tree.go index d94ee390c..ae7a829e5 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -25,16 +25,17 @@ func NewImmutableTree(db dbm.DB, cacheSize int) *ImmutableTree { return &ImmutableTree{ // NodeDB-backed Tree. // memDB created but should never be written to - ndb: newNodeDB(db, dbm.NewMemDB(), cacheSize, 1, 0), + ndb: newNodeDB(db, dbm.NewMemDB(), cacheSize, nil), } } -// NewImmutableTreePruningOpts creates ImmutableTree with specified pruning strategy. +// NewImmutableTreeWithOpts creates ImmutableTree with specified pruning/writing strategy. // Persists every `keepEvery` version to snapDB and saves last `keepRecent` versions to recentDB -func NewImmutableTreePruningOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *ImmutableTree { +// If sync is true, writes on nodeDB.Commit are blocking +func NewImmutableTreeWithOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, opts *Options) *ImmutableTree { return &ImmutableTree{ // NodeDB-backed Tree. - ndb: newNodeDB(snapDB, recentDB, cacheSize, keepEvery, keepRecent), + ndb: newNodeDB(snapDB, recentDB, cacheSize, opts), } } diff --git a/mutable_tree.go b/mutable_tree.go index 742be39a3..648c57e8c 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -27,12 +27,12 @@ type MutableTree struct { func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { // memDB is initialized but should never be written to memDB := dbm.NewMemDB() - return NewMutableTreePruningOpts(db, memDB, cacheSize, 1, 0) + return NewMutableTreeWithOpts(db, memDB, cacheSize, nil) } -// NewMutableTreePruningOpts returns a new tree with the specified cache size, datastores and pruning options -func NewMutableTreePruningOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *MutableTree { - ndb := newNodeDB(snapDB, recentDB, cacheSize, keepEvery, keepRecent) +// NewMutableTreeWithOpts returns a new tree with the specified cache size, datastores and options +func NewMutableTreeWithOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, opts *Options) *MutableTree { + ndb := newNodeDB(snapDB, recentDB, cacheSize, opts) head := &ImmutableTree{ndb: ndb} return &MutableTree{ diff --git a/nodedb.go b/nodedb.go index f3efa0cc9..36693ee69 100644 --- a/nodedb.go +++ b/nodedb.go @@ -38,8 +38,7 @@ type nodeDB struct { recentDB dbm.DB // Memory node storage. snapshotBatch dbm.Batch // Batched writing buffer. recentBatch dbm.Batch // Batched writing buffer for recentDB. - keepRecent int64 // number of recent versions to store in recentDB - keepEvery int64 // frequency of snaphots saved to disk + opts *Options // Options to customize for pruning/writing latestVersion int64 nodeCache map[string]*list.Element // Node cache. @@ -47,14 +46,16 @@ type nodeDB struct { nodeCacheQueue *list.List // LRU queue of cache elements. Used for deletion. } -func newNodeDB(snapshotDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, keepRecent int64) *nodeDB { +func newNodeDB(snapshotDB dbm.DB, recentDB dbm.DB, cacheSize int, opts *Options) *nodeDB { + if opts == nil { + opts = DefaultOptions() + } ndb := &nodeDB{ snapshotDB: snapshotDB, recentDB: recentDB, snapshotBatch: snapshotDB.NewBatch(), recentBatch: recentDB.NewBatch(), - keepRecent: keepRecent, - keepEvery: keepEvery, + opts: opts, latestVersion: 0, // initially invalid nodeCache: make(map[string]*list.Element), nodeCacheSize: cacheSize, @@ -64,11 +65,11 @@ func newNodeDB(snapshotDB dbm.DB, recentDB dbm.DB, cacheSize int, keepEvery, kee } func (ndb *nodeDB) isSnapshotVersion(version int64) bool { - return ndb.keepEvery != 0 && version%ndb.keepEvery == 0 + return ndb.opts.KeepEvery != 0 && version%ndb.opts.KeepEvery == 0 } func (ndb *nodeDB) isRecentVersion(version int64) bool { - return ndb.keepRecent != 0 && version > ndb.latestVersion-ndb.keepRecent + return ndb.opts.KeepRecent != 0 && version > ndb.latestVersion-ndb.opts.KeepRecent } // GetNode gets a node from memory or disk. If it is an inner node, it does not @@ -230,9 +231,9 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { for hash, fromVersion := range orphans { flushToDisk := false - if ndb.keepEvery != 0 { + if ndb.opts.KeepEvery != 0 { // if snapshot version in between fromVersion and toVersion INCLUSIVE, then flush to disk. - flushToDisk = fromVersion/ndb.keepEvery != toVersion/ndb.keepEvery || ndb.isSnapshotVersion(fromVersion) + flushToDisk = fromVersion/ndb.opts.KeepEvery != toVersion/ndb.opts.KeepEvery || ndb.isSnapshotVersion(fromVersion) } debug("SAVEORPHAN %v-%v %X flushToDisk: %t\n", fromVersion, toVersion, hash, flushToDisk) ndb.saveOrphan([]byte(hash), fromVersion, toVersion, flushToDisk) @@ -250,7 +251,7 @@ func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushTo } if flushToDisk { // save to disk with toVersion equal to snapshotVersion closest to original toVersion - snapVersion := toVersion - (toVersion % ndb.keepEvery) + snapVersion := toVersion - (toVersion % ndb.opts.KeepEvery) key := ndb.orphanKey(fromVersion, snapVersion, hash) ndb.snapshotBatch.Set(key, hash) } @@ -259,7 +260,7 @@ func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushTo // deleteOrphans deletes orphaned nodes from disk, and the associated orphan // entries. func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { - if ndb.keepRecent != 0 { + if ndb.opts.KeepRecent != 0 { ndb.deleteOrphansMem(version) } if ndb.isSnapshotVersion(version) && !memOnly { @@ -273,12 +274,12 @@ func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { func (ndb *nodeDB) deleteOrphansMem(version int64) { traverseOrphansVersionFromDB(ndb.recentDB, version, func(key, hash []byte) { - if ndb.keepRecent == 0 { + if ndb.opts.KeepRecent == 0 { return } ndb.recentBatch.Delete(key) // common case, we are deleting orphans from least recent version that is getting pruned from memDB - if version == ndb.latestVersion-ndb.keepRecent { + if version == ndb.latestVersion-ndb.opts.KeepRecent { // delete orphan look-up, delete and uncache node ndb.recentBatch.Delete(ndb.nodeKey(hash)) ndb.uncacheNode(hash) @@ -317,10 +318,10 @@ func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk b } func (ndb *nodeDB) PruneRecentVersions() (prunedVersions []int64) { - if ndb.keepRecent == 0 || ndb.latestVersion-ndb.keepRecent <= 0 { + if ndb.opts.KeepRecent == 0 || ndb.latestVersion-ndb.opts.KeepRecent <= 0 { return nil } - pruneVer := ndb.latestVersion - ndb.keepRecent + pruneVer := ndb.latestVersion - ndb.opts.KeepRecent ndb.DeleteVersionFromRecent(pruneVer, true) if ndb.isSnapshotVersion(pruneVer) { return nil @@ -503,12 +504,20 @@ func (ndb *nodeDB) Commit() { ndb.mtx.Lock() defer ndb.mtx.Unlock() - if ndb.keepEvery != 0 { - ndb.snapshotBatch.Write() + if ndb.opts.KeepEvery != 0 { + if ndb.opts.Sync { + ndb.snapshotBatch.WriteSync() + } else { + ndb.snapshotBatch.Write() + } ndb.snapshotBatch.Close() } - if ndb.keepRecent != 0 { - ndb.recentBatch.Write() + if ndb.opts.KeepRecent != 0 { + if ndb.opts.Sync { + ndb.recentBatch.WriteSync() + } else { + ndb.recentBatch.Write() + } ndb.recentBatch.Close() } ndb.snapshotBatch = ndb.snapshotDB.NewBatch() diff --git a/options.go b/options.go new file mode 100644 index 000000000..e2ba76bb8 --- /dev/null +++ b/options.go @@ -0,0 +1,36 @@ +package iavl + +// Define Options for customizing pruning/writing strategy of IAVL tree +type Options struct { + KeepEvery int64 + KeepRecent int64 + Sync bool +} + +// Returns the default options for IAVL +func DefaultOptions() *Options { + return &Options{ + KeepEvery: 1, + KeepRecent: 0, + Sync: false, + } +} + +// Return Options with given pruning strategy. Sync=false +func PruningOptions(keepEvery, keepRecent int64) *Options { + return &Options{ + KeepEvery: keepEvery, + KeepRecent: keepRecent, + Sync: false, + } +} + +// Return Options intended for benchmark tests +// with given pruning strategy. Sync = true +func BenchingOptions(keepEvery, keepRecent int64) *Options { + return &Options{ + KeepEvery: keepEvery, + KeepRecent: keepRecent, + Sync: true, + } +} diff --git a/pruning_test.go b/pruning_test.go index c1092a284..67a283b8b 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -28,7 +28,7 @@ func TestSave(t *testing.T) { keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (rand.Int63n(3) + 1) * 100 - mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) // create 1000 versions for i := 0; i < 1000; i++ { @@ -50,7 +50,7 @@ func TestSave(t *testing.T) { ver := int64(v) // check that version is supposed to exist given pruning strategy require.True(t, ver%keepEvery == 0 || mt.Version()-ver <= keepRecent, - "Version: %d should not exist. KeepEvery: %d, KeepRecent: %d", v, keepEvery, keepRecent) + "Version: %d should not exist. KeepEvery: %d, KeepRecent: %d", v, PruningOptions(keepEvery, keepRecent)) // check that root exists in nodeDB lv, err := mt.LazyLoadVersion(ver) @@ -60,10 +60,10 @@ func TestSave(t *testing.T) { // check all expected versions are available. for j := keepEvery; j <= mt.Version(); j += keepEvery { - require.True(t, mt.VersionExists(int64(j)), "Expected snapshot version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", j, keepEvery, keepRecent) + require.True(t, mt.VersionExists(int64(j)), "Expected snapshot version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", j, PruningOptions(keepEvery, keepRecent)) } for k := mt.Version() - keepRecent + 1; k <= mt.Version(); k++ { - require.True(t, mt.VersionExists(int64(k)), "Expected recent version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", k, keepEvery, keepRecent) + require.True(t, mt.VersionExists(int64(k)), "Expected recent version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", k, PruningOptions(keepEvery, keepRecent)) } // check that there only exists correct number of roots in nodeDB @@ -83,7 +83,7 @@ func TestDeleteOrphans(t *testing.T) { keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (rand.Int63n(3) + 1) * 100 - mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) // create 1200 versions (multiple of any possible snapshotting version) for i := 0; i < 1200; i++ { @@ -149,8 +149,8 @@ func TestDeleteOrphans(t *testing.T) { traverseOrphansFromDB(mt.ndb.recentDB, lastfn) require.Equal(t, 0, size, "Orphans still exist in recentDB") - require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "More nodes in recentDB than expected. KeepEvery: %d, KeepRecent: %d.", keepEvery, keepRecent) - require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "More nodes in snapshotDB than expected. KeepEvery: %d, KeepRecent: %d.", keepEvery, keepRecent) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "More nodes in recentDB than expected. KeepEvery: %d, KeepRecent: %d.", PruningOptions(keepEvery, keepRecent)) + require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "More nodes in snapshotDB than expected. KeepEvery: %d, KeepRecent: %d.", PruningOptions(keepEvery, keepRecent)) } func TestReplaceKeys(t *testing.T) { @@ -159,7 +159,7 @@ func TestReplaceKeys(t *testing.T) { keepRecent := int64(1) //keep 1 version in memDB keepEvery := int64(5) - mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) // Replace the same 10 keys with different values for i := 0; i < 10; i++ { @@ -195,7 +195,7 @@ func TestRemoveKeys(t *testing.T) { keepRecent := int64(1) //keep 1 version in memDB keepEvery := int64(10) - mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) for v := 0; v < 10; v++ { for i := 0; i < 10; i++ { @@ -228,7 +228,7 @@ func TestDBState(t *testing.T) { keepRecent := int64(5) keepEvery := int64(1) - mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) // create 5 versions for i := 0; i < 5; i++ { @@ -261,7 +261,7 @@ func TestSanity1(t *testing.T) { keepRecent := int64(1) keepEvery := int64(5) - mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) // create 5 versions for i := 0; i < 5; i++ { @@ -307,7 +307,7 @@ func TestSanity2(t *testing.T) { keepRecent := int64(1) keepEvery := int64(5) - mt := NewMutableTreePruningOpts(db, mdb, 0, keepEvery, keepRecent) + mt := NewMutableTreeWithOpts(db, mdb, 0, PruningOptions(keepEvery, keepRecent)) // create 5 versions for i := 0; i < 5; i++ { @@ -348,7 +348,7 @@ func TestSanity3(t *testing.T) { keepRecent := int64(4) keepEvery := int64(100) - mt := NewMutableTreePruningOpts(db, mdb, 5, keepEvery, keepRecent) + mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) // create 1000 versions numSnapNodes := 0 @@ -387,8 +387,8 @@ func TestNoSnapshots(t *testing.T) { db, mdb, close := getTestDBs() defer close() - keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB - mt := NewMutableTreePruningOpts(db, mdb, 5, 0, keepRecent) // test no snapshots + keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB + mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(0, keepRecent)) // test no snapshots for i := 0; i < 50; i++ { // set 5 keys per version @@ -424,10 +424,10 @@ func TestNoSnapshots(t *testing.T) { } func TestNoRecents(t *testing.T) { - db, mdb, close := getTestDBs() + db, _, close := getTestDBs() defer close() - mt := NewMutableTreePruningOpts(db, mdb, 5, 1, 0) + mt := NewMutableTree(db, 5) for i := 0; i < 50; i++ { // set 5 keys per version diff --git a/testutils_test.go b/testutils_test.go index 976f98f9b..864021870 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -33,8 +33,10 @@ func getTestTree(cacheSize int) *MutableTree { keepRecent := mrand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (mrand.Int63n(3) + 1) * 100 // snapshot every {100,200,300} versions + opts := PruningOptions(keepEvery, keepRecent) + // Use MemDB for recentDB and snapshotDB - return NewMutableTreePruningOpts(db.NewMemDB(), db.NewMemDB(), cacheSize, keepEvery, keepRecent) + return NewMutableTreeWithOpts(db.NewMemDB(), db.NewMemDB(), cacheSize, opts) } // Convenience for a new node From 6ed75ffa334da64522378321338271021abaf6f3 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 29 Jul 2019 10:49:51 -0700 Subject: [PATCH 34/57] use benching options in benchmark tests --- benchmarks/bench_test.go | 2 +- benchmarks/prune_test.go | 4 +--- go.sum | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index df16ec3ee..4a07877e2 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -21,7 +21,7 @@ func randBytes(length int) []byte { } func prepareTree(b *testing.B, snapdb db.DB, memdb db.DB, keepEvery int64, keepRecent int64, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { - t := iavl.NewMutableTreePruningOpts(snapdb, memdb, size, keepEvery, keepRecent) + t := iavl.NewMutableTreeWithOpts(snapdb, memdb, size, iavl.BenchingOptions(keepEvery, keepRecent)) keys := make([][]byte, size) for i := 0; i < size; i++ { diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index 5fc6cc4cc..ed8d58fdb 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -6,7 +6,6 @@ import ( "runtime" "testing" - "github.com/syndtr/goleveldb/opts" db "github.com/tendermint/tm-cmn/db" ) @@ -29,8 +28,7 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 runtime.GC() // always initialize tree with goleveldb as snapshotDB and memDB as recentDB - dbOptions := opts.Options{} - snapDB := NewGoLevelDBWithOpts("test", dirName, &dbOptions) + snapDB := db.NewDB("test", "goleveldb", dirName) defer snapDB.Close() // var mem runtime.MemStats diff --git a/go.sum b/go.sum index 5f682020a..df16c31df 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tendermint/go-amino v0.14.1 h1:o2WudxNfdLNBwMyl2dqOJxiro5rfrEaU0Ugs6offJMk= From 1a5311fc732a7474f861e4d522bad195eb8b9c7e Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 30 Jul 2019 16:01:33 +0000 Subject: [PATCH 35/57] Apply suggestions from fede code review Co-Authored-By: Federico Kunze <31522760+fedekunze@users.noreply.github.com> --- PRUNING.md | 8 +++++--- options.go | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/PRUNING.md b/PRUNING.md index 9261aa498..0fa5e2855 100644 --- a/PRUNING.md +++ b/PRUNING.md @@ -9,7 +9,8 @@ We can set custom pruning fields in IAVL using: `NewMutableTreePruningOpts` ### NodeDB NodeDB has extra fields: -``` + +```go recentDB dbm.DB // Memory node storage. recentBatch dbm.Batch // Batched writing buffer for memDB. @@ -22,7 +23,8 @@ If version is not going to be persisted to disk, the version is simply saved in If version is persisted to disk, the version is written to `recentDB` **and** `snapshotDB` (typically `levelDB`) #### Orphans: -Save orphan to memDB under `o|toVersion|fromVersion`. + +Save orphan to `memDB` under `o|toVersion|fromVersion`. If there exists snapshot version `snapVersion` s.t. `fromVersion < snapVersion < toVersion`, save orphan to disk as well under `o|snapVersion|fromVersion`. NOTE: in unlikely event, that two snapshot versions exist between `fromVersion` and `toVersion`, we use closest snapshot version that is less than `toVersion` @@ -33,4 +35,4 @@ Can then simply use the old delete algorithm with some minor simplifications/opt MutableTree can be instantiated with a pruning-aware NodeDB. -When `MutableTree` saves a new Version, it also calls `PruneRecentVersions` on nodeDB which causes oldest version in recentDB (`latestVersion - keepRecent`) to get pruned. \ No newline at end of file +When `MutableTree` saves a new Version, it also calls `PruneRecentVersions` on nodeDB which causes oldest version in recentDB (`latestVersion - keepRecent`) to get pruned. diff --git a/options.go b/options.go index e2ba76bb8..6671b16c8 100644 --- a/options.go +++ b/options.go @@ -1,13 +1,13 @@ package iavl -// Define Options for customizing pruning/writing strategy of IAVL tree +// Options define customized pruning/writing strategies for the IAVL tree type Options struct { KeepEvery int64 KeepRecent int64 Sync bool } -// Returns the default options for IAVL +// DefaultOptions returns the default options for IAVL func DefaultOptions() *Options { return &Options{ KeepEvery: 1, @@ -25,7 +25,7 @@ func PruningOptions(keepEvery, keepRecent int64) *Options { } } -// Return Options intended for benchmark tests +// BenchingOptions returns Options intended for benchmark tests // with given pruning strategy. Sync = true func BenchingOptions(keepEvery, keepRecent int64) *Options { return &Options{ From 54b2dacd30682b5063491042f020d99d163846e5 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Tue, 30 Jul 2019 09:06:42 -0700 Subject: [PATCH 36/57] tidy and remove unnecessary orphan --- benchmarks/prune_test.go | 11 ++++++++--- go.sum | 1 - mutable_tree.go | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index ed8d58fdb..104594adc 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -40,7 +40,7 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 b.ResetTimer() t, _ := prepareTree(b, snapDB, db.NewMemDB(), keepEvery, keepRecent, 5, keyLen, dataLen) // create 30000 versions - for i := 0; i < b.N; i++ { + for i := 0; i < 5000; i++ { // create 5 keys per version for j := 0; j < 5; j++ { t.Set(randBytes(keyLen), randBytes(dataLen)) @@ -59,6 +59,9 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 // maxVersion = i // } // b.StartTimer() + b.StopTimer() + runtime.GC() + b.StartTimer() } //fmt.Printf("Maxmimum Memory usage was %0.2f MB at height %d\n", float64(memSize)/1000000, maxVersion) b.StopTimer() @@ -66,8 +69,10 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 func BenchmarkPruningStrategies(b *testing.B) { ps := []pruningstrat{ - {1, 0}, // default pruning strategy - {0, 1}, // keep single recent version + {1, 0}, // default pruning strategy + //{1, 1}, + {0, 1}, // keep single recent version + {100, 1}, {100, 5}, // simple pruning // {1000, 10}, // average pruning // {1000, 1}, // extreme pruning diff --git a/go.sum b/go.sum index df16c31df..5f682020a 100644 --- a/go.sum +++ b/go.sum @@ -98,7 +98,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tendermint/go-amino v0.14.1 h1:o2WudxNfdLNBwMyl2dqOJxiro5rfrEaU0Ugs6offJMk= diff --git a/mutable_tree.go b/mutable_tree.go index 648c57e8c..628dabd21 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -211,7 +211,6 @@ func (tree *MutableTree) recursiveRemove(node *Node, key []byte) ([]byte, *Node, } orphaned = append(orphaned, node) if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed - orphaned = append(orphaned, node) return node.rightHash, node.rightNode, node.key, value, orphaned } From f1e117d590641e0b38ba8a165231bcaf7f902697 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Tue, 30 Jul 2019 09:08:30 -0700 Subject: [PATCH 37/57] complete fede doc comments --- options.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/options.go b/options.go index 6671b16c8..f0f38ab36 100644 --- a/options.go +++ b/options.go @@ -16,7 +16,8 @@ func DefaultOptions() *Options { } } -// Return Options with given pruning strategy. Sync=false +// PruningOptions returns Options with a given pruning strategy. +// Sync is set to false func PruningOptions(keepEvery, keepRecent int64) *Options { return &Options{ KeepEvery: keepEvery, @@ -26,7 +27,7 @@ func PruningOptions(keepEvery, keepRecent int64) *Options { } // BenchingOptions returns Options intended for benchmark tests -// with given pruning strategy. Sync = true +// with a given pruning strategy. Sync is set to true func BenchingOptions(keepEvery, keepRecent int64) *Options { return &Options{ KeepEvery: keepEvery, From 1e9bd2cee60936b8466702d78c7057fe3930c26e Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Wed, 31 Jul 2019 11:40:45 -0700 Subject: [PATCH 38/57] fix linter --- go.mod | 3 +++ go.sum | 9 +++++++++ mutable_tree_test.go | 3 ++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c7d712fcf..e262b8d8e 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,13 @@ module github.com/tendermint/iavl go 1.12 require ( + github.com/kr/pretty v0.1.0 // indirect github.com/pkg/errors v0.8.1 github.com/stretchr/testify v1.3.0 github.com/tendermint/go-amino v0.14.1 github.com/tendermint/tendermint v0.32.1 github.com/tendermint/tm-cmn v0.0.0-20190716080004-dfcde30d5acb golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a + golang.org/x/sys v0.0.0-20190312061237-fead79001313 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 5f682020a..af6be0790 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,11 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -127,6 +132,8 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUk golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -141,6 +148,8 @@ google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/mutable_tree_test.go b/mutable_tree_test.go index cbf683049..ea272b04c 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -21,7 +21,7 @@ func TestDelete(t *testing.T) { require.NoError(t, tree.DeleteVersion(version)) - k1Value, _, err := tree.GetVersionedWithProof([]byte("k1"), version) + k1Value, _, _ := tree.GetVersionedWithProof([]byte("k1"), version) require.Nil(t, k1Value) key := tree.ndb.rootKey(version) @@ -29,6 +29,7 @@ func TestDelete(t *testing.T) { tree.versions[version] = true k1Value, _, err = tree.GetVersionedWithProof([]byte("k1"), version) + require.Nil(t, err) require.Equal(t, 0, bytes.Compare([]byte("Fred"), k1Value)) } From 9e8fde7055a1f0d8f6c04a6a5c498ae816f8b7d5 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 5 Aug 2019 12:14:07 -0700 Subject: [PATCH 39/57] replace keys pruning tests --- benchmarks/bench_test.go | 2 +- benchmarks/prune_test.go | 13 +- .../results/Adityas-MBP-pruning-results.txt | 9 ++ benchmarks/results/Adityas-MBP-results.txt | 147 ++++++++++++++++++ go.sum | 2 - 5 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 benchmarks/results/Adityas-MBP-pruning-results.txt create mode 100644 benchmarks/results/Adityas-MBP-results.txt diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 4e29260a6..bbefeffc0 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -21,7 +21,7 @@ func randBytes(length int) []byte { } func prepareTree(b *testing.B, snapdb db.DB, memdb db.DB, keepEvery int64, keepRecent int64, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { - t := iavl.NewMutableTreeWithOpts(snapdb, memdb, size, iavl.BenchingOptions(keepEvery, keepRecent)) + t := iavl.NewMutableTreeWithOpts(snapdb, memdb, size, iavl.PruningOptions(keepEvery, keepRecent)) keys := make([][]byte, size) for i := 0; i < size; i++ { diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index 104594adc..9f6b500ba 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -2,11 +2,12 @@ package benchmarks import ( "fmt" + "math/rand" "os" "runtime" "testing" - db "github.com/tendermint/tm-cmn/db" + db "github.com/tendermint/tm-db" ) type pruningstrat struct { @@ -35,15 +36,21 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 // runtime.ReadMemStats(&mem) // memSize := mem.Alloc // maxVersion := 0 + var keys [][]byte + for i := 0; i < 100; i++ { + keys = append(keys, randBytes(keyLen)) + } // reset timer after initialization logic b.ResetTimer() t, _ := prepareTree(b, snapDB, db.NewMemDB(), keepEvery, keepRecent, 5, keyLen, dataLen) + // create 30000 versions for i := 0; i < 5000; i++ { - // create 5 keys per version + // set 5 keys per version for j := 0; j < 5; j++ { - t.Set(randBytes(keyLen), randBytes(dataLen)) + index := rand.Int63n(100) + t.Set(keys[index], randBytes(dataLen)) } _, _, err := t.SaveVersion() if err != nil { diff --git a/benchmarks/results/Adityas-MBP-pruning-results.txt b/benchmarks/results/Adityas-MBP-pruning-results.txt new file mode 100644 index 000000000..f601ba7a9 --- /dev/null +++ b/benchmarks/results/Adityas-MBP-pruning-results.txt @@ -0,0 +1,9 @@ +goos: darwin +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkPruningStrategies/PruningStrategy{1-0}-KeyLen:16-DataLen:40-8 1 2837806322 ns/op +BenchmarkPruningStrategies/PruningStrategy{0-1}-KeyLen:16-DataLen:40-8 1 1124373981 ns/op +BenchmarkPruningStrategies/PruningStrategy{100-1}-KeyLen:16-DataLen:40-8 1 1255040658 ns/op +BenchmarkPruningStrategies/PruningStrategy{100-5}-KeyLen:16-DataLen:40-8 1 1459752743 ns/op +PASS +ok github.com/tendermint/iavl/benchmarks 12.375s diff --git a/benchmarks/results/Adityas-MBP-results.txt b/benchmarks/results/Adityas-MBP-results.txt new file mode 100644 index 000000000..341c236d1 --- /dev/null +++ b/benchmarks/results/Adityas-MBP-results.txt @@ -0,0 +1,147 @@ +cd benchmarks && \ + go test -bench=RandomBytes . && \ + go test -bench=Small . && \ + go test -bench=Medium . && \ + go test -bench=BenchmarkMemKeySizes . +goos: darwin +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkRandomBytes/random-4-8 30000000 50.1 ns/op +BenchmarkRandomBytes/random-16-8 20000000 77.4 ns/op +BenchmarkRandomBytes/random-32-8 20000000 110 ns/op +BenchmarkRandomBytes/random-100-8 5000000 261 ns/op +BenchmarkRandomBytes/random-1000-8 1000000 2125 ns/op +PASS +ok github.com/tendermint/iavl/benchmarks 9.239s +Init Tree took 0.55 MB +goos: darwin +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkSmall/memdb-1-0-1000-100-4-10/query-miss-8 1000000 2531 ns/op +BenchmarkSmall/memdb-1-0-1000-100-4-10/query-hits-8 500000 3429 ns/op +BenchmarkSmall/memdb-1-0-1000-100-4-10/update-8 10000 116557 ns/op +BenchmarkSmall/memdb-1-0-1000-100-4-10/block-8 100 33767094 ns/op +Init Tree took 0.14 MB +BenchmarkSmall/goleveldb-1-0-1000-100-4-10/query-miss-8 500000 3982 ns/op +BenchmarkSmall/goleveldb-1-0-1000-100-4-10/query-hits-8 300000 5113 ns/op +BenchmarkSmall/goleveldb-1-0-1000-100-4-10/update-8 20000 83604 ns/op +BenchmarkSmall/goleveldb-1-0-1000-100-4-10/block-8 100 14777165 ns/op +Init Tree took 0.58 MB +BenchmarkSmall/memdb-0-1-1000-100-4-10/query-miss-8 500000 2367 ns/op +BenchmarkSmall/memdb-0-1-1000-100-4-10/query-hits-8 500000 2808 ns/op +BenchmarkSmall/memdb-0-1-1000-100-4-10/update-8 50000 33475 ns/op +BenchmarkSmall/memdb-0-1-1000-100-4-10/block-8 300 15409232 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/goleveldb-0-1-1000-100-4-10/query-miss-8 500000 2608 ns/op +BenchmarkSmall/goleveldb-0-1-1000-100-4-10/query-hits-8 500000 3063 ns/op +BenchmarkSmall/goleveldb-0-1-1000-100-4-10/update-8 50000 37384 ns/op +BenchmarkSmall/goleveldb-0-1-1000-100-4-10/block-8 200 13892236 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/memdb-100-5-1000-100-4-10/query-miss-8 500000 2794 ns/op +BenchmarkSmall/memdb-100-5-1000-100-4-10/query-hits-8 500000 3108 ns/op +BenchmarkSmall/memdb-100-5-1000-100-4-10/update-8 30000 46381 ns/op +BenchmarkSmall/memdb-100-5-1000-100-4-10/block-8 200 15793077 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/goleveldb-100-5-1000-100-4-10/query-miss-8 1000000 2581 ns/op +BenchmarkSmall/goleveldb-100-5-1000-100-4-10/query-hits-8 500000 2971 ns/op +BenchmarkSmall/goleveldb-100-5-1000-100-4-10/update-8 30000 54246 ns/op +BenchmarkSmall/goleveldb-100-5-1000-100-4-10/block-8 200 17805252 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/memdb-1000-10-1000-100-4-10/query-miss-8 500000 2515 ns/op +BenchmarkSmall/memdb-1000-10-1000-100-4-10/query-hits-8 500000 2713 ns/op +BenchmarkSmall/memdb-1000-10-1000-100-4-10/update-8 30000 56000 ns/op +BenchmarkSmall/memdb-1000-10-1000-100-4-10/block-8 200 18247795 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/goleveldb-1000-10-1000-100-4-10/query-miss-8 500000 2406 ns/op +BenchmarkSmall/goleveldb-1000-10-1000-100-4-10/query-hits-8 500000 2716 ns/op +BenchmarkSmall/goleveldb-1000-10-1000-100-4-10/update-8 30000 54957 ns/op +BenchmarkSmall/goleveldb-1000-10-1000-100-4-10/block-8 100 10522437 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/memdb-1000-1-1000-100-4-10/query-miss-8 500000 2560 ns/op +BenchmarkSmall/memdb-1000-1-1000-100-4-10/query-hits-8 500000 3053 ns/op +BenchmarkSmall/memdb-1000-1-1000-100-4-10/update-8 50000 36769 ns/op +BenchmarkSmall/memdb-1000-1-1000-100-4-10/block-8 200 14357739 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/goleveldb-1000-1-1000-100-4-10/query-miss-8 500000 3290 ns/op +BenchmarkSmall/goleveldb-1000-1-1000-100-4-10/query-hits-8 500000 3437 ns/op +BenchmarkSmall/goleveldb-1000-1-1000-100-4-10/update-8 50000 56914 ns/op +BenchmarkSmall/goleveldb-1000-1-1000-100-4-10/block-8 100 15012439 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/memdb-10000-100-1000-100-4-10/query-miss-8 500000 2850 ns/op +BenchmarkSmall/memdb-10000-100-1000-100-4-10/query-hits-8 500000 2730 ns/op +BenchmarkSmall/memdb-10000-100-1000-100-4-10/update-8 10000 103250 ns/op +BenchmarkSmall/memdb-10000-100-1000-100-4-10/block-8 50 39550503 ns/op +Init Tree took 0.55 MB +BenchmarkSmall/goleveldb-10000-100-1000-100-4-10/query-miss-8 500000 2684 ns/op +BenchmarkSmall/goleveldb-10000-100-1000-100-4-10/query-hits-8 500000 3280 ns/op +BenchmarkSmall/goleveldb-10000-100-1000-100-4-10/update-8 10000 107483 ns/op +BenchmarkSmall/goleveldb-10000-100-1000-100-4-10/block-8 50 40069720 ns/op +PASS +ok github.com/tendermint/iavl/benchmarks 100.688s +Init Tree took 48.86 MB +goos: darwin +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkMedium/memdb-1-0-100000-100-16-40/query-miss-8 200000 6281 ns/op +BenchmarkMedium/memdb-1-0-100000-100-16-40/query-hits-8 200000 7490 ns/op +BenchmarkMedium/memdb-1-0-100000-100-16-40/update-8 5000 589147 ns/op +BenchmarkMedium/memdb-1-0-100000-100-16-40/block-8 20 69479350 ns/op +Init Tree took 10.91 MB +BenchmarkMedium/goleveldb-1-0-100000-100-16-40/query-miss-8 50000 22021 ns/op +BenchmarkMedium/goleveldb-1-0-100000-100-16-40/query-hits-8 50000 27466 ns/op +BenchmarkMedium/goleveldb-1-0-100000-100-16-40/update-8 10000 374747 ns/op +BenchmarkMedium/goleveldb-1-0-100000-100-16-40/block-8 50 47548357 ns/op +Init Tree took 48.81 MB +BenchmarkMedium/memdb-0-1-100000-100-16-40/query-miss-8 200000 6472 ns/op +BenchmarkMedium/memdb-0-1-100000-100-16-40/query-hits-8 200000 8117 ns/op +BenchmarkMedium/memdb-0-1-100000-100-16-40/update-8 1000 1413643 ns/op +BenchmarkMedium/memdb-0-1-100000-100-16-40/block-8 20 119165460 ns/op +Init Tree took 48.87 MB +BenchmarkMedium/goleveldb-0-1-100000-100-16-40/query-miss-8 200000 6465 ns/op +BenchmarkMedium/goleveldb-0-1-100000-100-16-40/query-hits-8 200000 7681 ns/op +BenchmarkMedium/goleveldb-0-1-100000-100-16-40/update-8 2000 853301 ns/op +BenchmarkMedium/goleveldb-0-1-100000-100-16-40/block-8 20 100658873 ns/op +Init Tree took 48.85 MB +BenchmarkMedium/memdb-100-5-100000-100-16-40/query-miss-8 200000 6710 ns/op +BenchmarkMedium/memdb-100-5-100000-100-16-40/query-hits-8 200000 7424 ns/op +BenchmarkMedium/memdb-100-5-100000-100-16-40/update-8 5000 878736 ns/op +BenchmarkMedium/memdb-100-5-100000-100-16-40/block-8 20 105546937 ns/op +Init Tree took 48.84 MB +BenchmarkMedium/goleveldb-100-5-100000-100-16-40/query-miss-8 200000 6818 ns/op +BenchmarkMedium/goleveldb-100-5-100000-100-16-40/query-hits-8 200000 7820 ns/op +BenchmarkMedium/goleveldb-100-5-100000-100-16-40/update-8 5000 855615 ns/op +BenchmarkMedium/goleveldb-100-5-100000-100-16-40/block-8 20 90191776 ns/op +Init Tree took 48.86 MB +BenchmarkMedium/memdb-1000-10-100000-100-16-40/query-miss-8 200000 6628 ns/op +BenchmarkMedium/memdb-1000-10-100000-100-16-40/query-hits-8 200000 8083 ns/op +BenchmarkMedium/memdb-1000-10-100000-100-16-40/update-8 3000 900956 ns/op +BenchmarkMedium/memdb-1000-10-100000-100-16-40/block-8 20 93776159 ns/op +Init Tree took 48.87 MB +BenchmarkMedium/goleveldb-1000-10-100000-100-16-40/query-miss-8 200000 6316 ns/op +BenchmarkMedium/goleveldb-1000-10-100000-100-16-40/query-hits-8 200000 7373 ns/op +BenchmarkMedium/goleveldb-1000-10-100000-100-16-40/update-8 3000 810655 ns/op +BenchmarkMedium/goleveldb-1000-10-100000-100-16-40/block-8 20 89698234 ns/op +Init Tree took 48.83 MB +BenchmarkMedium/memdb-1000-1-100000-100-16-40/query-miss-8 200000 6414 ns/op +BenchmarkMedium/memdb-1000-1-100000-100-16-40/query-hits-8 200000 7273 ns/op +BenchmarkMedium/memdb-1000-1-100000-100-16-40/update-8 2000 793667 ns/op +BenchmarkMedium/memdb-1000-1-100000-100-16-40/block-8 20 83505306 ns/op +Init Tree took 48.88 MB +BenchmarkMedium/goleveldb-1000-1-100000-100-16-40/query-miss-8 200000 6258 ns/op +BenchmarkMedium/goleveldb-1000-1-100000-100-16-40/query-hits-8 200000 7083 ns/op +BenchmarkMedium/goleveldb-1000-1-100000-100-16-40/update-8 2000 799026 ns/op +BenchmarkMedium/goleveldb-1000-1-100000-100-16-40/block-8 20 82225915 ns/op +Init Tree took 48.88 MB +BenchmarkMedium/memdb-10000-100-100000-100-16-40/query-miss-8 200000 6420 ns/op +BenchmarkMedium/memdb-10000-100-100000-100-16-40/query-hits-8 200000 7216 ns/op +BenchmarkMedium/memdb-10000-100-100000-100-16-40/update-8 5000 595187 ns/op +BenchmarkMedium/memdb-10000-100-100000-100-16-40/block-8 20 76132336 ns/op +Init Tree took 48.86 MB +BenchmarkMedium/goleveldb-10000-100-100000-100-16-40/query-miss-8 200000 10622 ns/op +BenchmarkMedium/goleveldb-10000-100-100000-100-16-40/query-hits-8 100000 16061 ns/op +BenchmarkMedium/goleveldb-10000-100-100000-100-16-40/update-8 2000 734238 ns/op +BenchmarkMedium/goleveldb-10000-100-100000-100-16-40/block-8 20 72135954 ns/op +PASS +ok github.com/tendermint/iavl/benchmarks 124.410s +PASS +ok github.com/tendermint/iavl/benchmarks 0.015s diff --git a/go.sum b/go.sum index ab203545a..6ede9566d 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,6 @@ github.com/tendermint/go-amino v0.14.1 h1:o2WudxNfdLNBwMyl2dqOJxiro5rfrEaU0Ugs6o github.com/tendermint/go-amino v0.14.1/go.mod h1:i/UKE5Uocn+argJJBb12qTZsCDBcAYMbR92AaJVmKso= github.com/tendermint/tendermint v0.32.1 h1:J8ddXMbCmG6GZjdCl/N1wgdXDU9uO91J2Y5CA9xYfGo= github.com/tendermint/tendermint v0.32.1/go.mod h1:jmPDAKuNkev9793/ivn/fTBnfpA9mGBww8MPRNPNxnU= -github.com/tendermint/tm-db v0.0.0-20190731085305-94017c88bf1d h1:yCHL2COLGLNfb4sA9AlzIHpapb8UATvAQyJulS6Eg6Q= -github.com/tendermint/tm-db v0.0.0-20190731085305-94017c88bf1d/go.mod h1:0cPKWu2Mou3IlxecH+MEUSYc1Ch537alLe6CpFrKzgw= github.com/tendermint/tm-db v0.1.1 h1:G3Xezy3sOk9+ekhjZ/kjArYIs1SmwV+1OUgNkj7RgV0= github.com/tendermint/tm-db v0.1.1/go.mod h1:0cPKWu2Mou3IlxecH+MEUSYc1Ch537alLe6CpFrKzgw= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= From fcc92e43e04b864327ce4ad8d1f4264ec4116807 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 5 Aug 2019 14:42:40 -0700 Subject: [PATCH 40/57] fix imports --- mutable_tree_test.go | 2 +- pruning_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mutable_tree_test.go b/mutable_tree_test.go index ea272b04c..fe41f71a8 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "github.com/tendermint/tm-cmn/db" + db "github.com/tendermint/tm-db" ) func TestDelete(t *testing.T) { diff --git a/pruning_test.go b/pruning_test.go index 67a283b8b..9fd44058c 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -8,8 +8,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "github.com/tendermint/tm-cmn/db" -) + db "github.com/tendermint/tm-db") func getTestDBs() (db.DB, db.DB, func()) { d, err := db.NewGoLevelDB("test", ".") From 7e10d5d8b3636413f663fe8ed1c72801f3c4d643 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 5 Aug 2019 22:06:13 +0000 Subject: [PATCH 41/57] Update pruning_test.go Co-Authored-By: Bot from GolangCI <42910462+golangcibot@users.noreply.github.com> --- pruning_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pruning_test.go b/pruning_test.go index 9fd44058c..f306c7853 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -8,7 +8,8 @@ import ( "testing" "github.com/stretchr/testify/require" - db "github.com/tendermint/tm-db") + db "github.com/tendermint/tm-db" +) func getTestDBs() (db.DB, db.DB, func()) { d, err := db.NewGoLevelDB("test", ".") From b15162c4cd723b57b0a69149ec66c93205bea2fe Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Tue, 6 Aug 2019 14:39:08 -0700 Subject: [PATCH 42/57] add DO results --- benchmarks/results/DigitalOcean-4vcpu-8gb-160gb.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 benchmarks/results/DigitalOcean-4vcpu-8gb-160gb.txt diff --git a/benchmarks/results/DigitalOcean-4vcpu-8gb-160gb.txt b/benchmarks/results/DigitalOcean-4vcpu-8gb-160gb.txt new file mode 100644 index 000000000..4c56f69c6 --- /dev/null +++ b/benchmarks/results/DigitalOcean-4vcpu-8gb-160gb.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/tendermint/iavl/benchmarks +BenchmarkPruningStrategies/PruningStrategy{1-0}-KeyLen:16-DataLen:40-4 1 3476623971 ns/op +BenchmarkPruningStrategies/PruningStrategy{0-1}-KeyLen:16-DataLen:40-4 1 2103728119 ns/op +BenchmarkPruningStrategies/PruningStrategy{100-1}-KeyLen:16-DataLen:40-4 1 2289531671 ns/op +BenchmarkPruningStrategies/PruningStrategy{100-5}-KeyLen:16-DataLen:40-4 1 2772934060 ns/op +PASS +ok github.com/tendermint/iavl/benchmarks 21.531s From 6367c2f5c579a9d4ca23b070ca6d528437de6b0e Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Tue, 6 Aug 2019 17:05:57 -0700 Subject: [PATCH 43/57] more pruning strategies --- benchmarks/prune_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index 9f6b500ba..1f3f17e01 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -81,6 +81,9 @@ func BenchmarkPruningStrategies(b *testing.B) { {0, 1}, // keep single recent version {100, 1}, {100, 5}, // simple pruning + {5, 1}, + {5, 2}, + {10, 2}, // {1000, 10}, // average pruning // {1000, 1}, // extreme pruning // {10000, 100}, // SDK pruning From 1772b7e4b5aaf9de1f9f3125ce60ef5c6b8136cb Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Fri, 9 Aug 2019 15:53:50 -0700 Subject: [PATCH 44/57] fix deleteVersionsFrom --- mutable_tree.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index 4b8ea79f1..dc7114c46 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -478,9 +478,6 @@ func (tree *MutableTree) deleteVersionsFrom(version int64) error { if version == tree.version { return errors.Errorf("cannot delete latest saved version (%d)", version) } - if _, ok := tree.versions[version]; !ok { - return errors.Wrap(ErrVersionDoesNotExist, "") - } tree.ndb.DeleteVersion(version, false) delete(tree.versions, version) } From 0fe8a808ebd6f231f06de6b677a9920ecbb22a13 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 12 Aug 2019 17:07:37 -0700 Subject: [PATCH 45/57] add sdk benchmark results --- .../158-pruning/sdk-pruning-1-0-results.txt | 164 ++++++++++++++++++ .../158-pruning/sdk-pruning-10-2-results.txt | 163 +++++++++++++++++ .../158-pruning/sdk-pruning-100-2-results.txt | 163 +++++++++++++++++ .../sdk-pruning-1000-5-results.txt | 162 +++++++++++++++++ .../158-pruning/sdk-pruning-5-2-results.txt | 163 +++++++++++++++++ 5 files changed, 815 insertions(+) create mode 100644 benchmarks/results/158-pruning/sdk-pruning-1-0-results.txt create mode 100644 benchmarks/results/158-pruning/sdk-pruning-10-2-results.txt create mode 100644 benchmarks/results/158-pruning/sdk-pruning-100-2-results.txt create mode 100644 benchmarks/results/158-pruning/sdk-pruning-1000-5-results.txt create mode 100644 benchmarks/results/158-pruning/sdk-pruning-5-2-results.txt diff --git a/benchmarks/results/158-pruning/sdk-pruning-1-0-results.txt b/benchmarks/results/158-pruning/sdk-pruning-1-0-results.txt new file mode 100644 index 000000000..99c419934 --- /dev/null +++ b/benchmarks/results/158-pruning/sdk-pruning-1-0-results.txt @@ -0,0 +1,164 @@ +Starting SimulateFromSeed with randomness created with seed 53 +Randomized simulation params: +{ + "PastEvidenceFraction": 0.5380706684387375, + "NumKeys": 130, + "EvidenceFraction": 0.38121722232625455, + "InitialLivenessWeightings": [ + 45, + 1, + 6 + ], + "LivenessTransitionMatrix": {}, + "BlockSizeTransitionMatrix": {} +} +Selected randomly generated parameters for simulated genesis: +{ + stake_per_account: "714557167989", + initially_bonded_validators: "130" +} +Selected randomly generated auth parameters: +{ + "max_memo_characters": "176", + "tx_sig_limit": "6", + "tx_size_cost_per_byte": "14", + "sig_verify_cost_ed25519": "754", + "sig_verify_cost_secp256k1": "719" +} +Selected randomly generated bank parameters: +{ + "send_enabled": true +} +Generated supply parameters: +{ + "supply": [ + { + "denom": "stake", + "amount": "185784863677140" + } + ] +} +Selected randomly generated governance parameters: +{ + "starting_proposal_id": "31", + "deposits": null, + "votes": null, + "proposals": null, + "deposit_params": { + "min_deposit": [ + { + "denom": "stake", + "amount": "496" + } + ], + "max_deposit_period": "11146000000000" + }, + "voting_params": { + "voting_period": "11146000000000" + }, + "tally_params": { + "quorum": "0.348000000000000000", + "threshold": "0.495000000000000000", + "veto": "0.288000000000000000" + } +} +Selected randomly generated minting parameters: +{ + "mint_denom": "stake", + "inflation_rate_change": "0.560000000000000000", + "inflation_max": "0.200000000000000000", + "inflation_min": "0.070000000000000000", + "goal_bonded": "0.670000000000000000", + "blocks_per_year": "6311520" +} +Selected randomly generated distribution parameters: +{ + "fee_pool": { + "community_pool": [] + }, + "community_tax": "0.090000000000000000", + "base_proposer_reward": "0.190000000000000000", + "bonus_proposer_reward": "0.100000000000000000", + "withdraw_addr_enabled": false, + "delegator_withdraw_infos": null, + "previous_proposer": "", + "outstanding_rewards": null, + "validator_accumulated_commissions": null, + "validator_historical_rewards": null, + "validator_current_rewards": null, + "delegator_starting_infos": null, + "validator_slash_events": null +} +Selected randomly generated staking parameters: +{ + "unbonding_time": "163286000000000", + "max_validators": 175, + "max_entries": 7, + "bond_denom": "stake" +} +Selected randomly generated slashing parameters: +{ + "max_evidence_age": "163286000000000", + "signed_blocks_window": "695", + "min_signed_per_window": "0.500000000000000000", + "downtime_jail_duration": "14486000000000", + "slash_fraction_double_sign": "0.024390243902439024", + "slash_fraction_downtime": "0.008130081300813008" +} +Starting the simulation from time Wed Aug 20 16:59:49 UTC 2566 (unixtime 18828003589) + +Simulating... block 8063/10000, operation 0/349. + +Simulation stopped early as all validators have been unbonded; nobody left to propose a block! +Event statistics: + /no-operation/failure => 39403 + auth/deduct_fee/failure => 232 + auth/deduct_fee/ok => 3501 + bank/multisend/failure => 388 + bank/multisend/ok => 5802 + bank/send/failure => 3773 + bank/send/ok => 59815 + beginblock/evidence => 4927 + beginblock/signing/missed => 2741 + beginblock/signing/signed => 19978 + distribution/set_withdraw_address/failure => 31822 + distribution/withdraw_delegator_reward/failure => 8225 + distribution/withdraw_delegator_reward/ok => 23820 + distribution/withdraw_validator_commission/failure => 30578 + distribution/withdraw_validator_commission/ok => 1213 + endblock/validatorupdates/added => 35 + endblock/validatorupdates/kicked => 165 + endblock/validatorupdates/updated => 3909 + gov/deposit/failure => 63464 + gov/deposit/ok => 341 + gov/submit_proposal/failure => 2002 + gov/submit_proposal/ok => 7442 + slashing/unjail/failure => 63077 + slashing/unjail/ok => 29 + staking/begin_redelegate/failure => 32582 + staking/begin_redelegate/ok => 21364 + staking/begin_unbonding/failure => 5 + staking/begin_unbonding/ok => 53397 + staking/create_validator/failure => 53883 + staking/create_validator/ok => 6 + staking/delegate/ok => 53694 + staking/edit_validator/failure => 380 + staking/edit_validator/ok => 2882 +GoLevelDB Stats +Compactions + Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +-------+------------+---------------+---------------+---------------+--------------- + 0 | 0 | 0.00000 | 19.52257 | 0.00000 | 1409.10107 + 1 | 78 | 100.56797 | 956.69678 | 49073.19592 | 49085.70664 + 2 | 542 | 1000.69145 | 1425.27505 | 112484.45822 | 112493.21130 + 3 | 166 | 329.11150 | 15.28762 | 1024.58862 | 1024.59469 +-------+------------+---------------+---------------+---------------+--------------- + Total | 786 | 1430.37092 | 2416.78201 | 162582.24277 | 164012.61369 + +GoLevelDB cached block size 8373420 +goos: linux +goarch: amd64 +pkg: github.com/cosmos/cosmos-sdk/simapp +BenchmarkFullAppSimulation-4 1 2925903867806 ns/op 359678535952 B/op 6131788601 allocs/op +PASS +ok github.com/cosmos/cosmos-sdk/simapp 2926.999s diff --git a/benchmarks/results/158-pruning/sdk-pruning-10-2-results.txt b/benchmarks/results/158-pruning/sdk-pruning-10-2-results.txt new file mode 100644 index 000000000..781197b93 --- /dev/null +++ b/benchmarks/results/158-pruning/sdk-pruning-10-2-results.txt @@ -0,0 +1,163 @@ +Starting SimulateFromSeed with randomness created with seed 53 +Randomized simulation params: +{ + "PastEvidenceFraction": 0.5380706684387375, + "NumKeys": 130, + "EvidenceFraction": 0.38121722232625455, + "InitialLivenessWeightings": [ + 45, + 1, + 6 + ], + "LivenessTransitionMatrix": {}, + "BlockSizeTransitionMatrix": {} +} +Selected randomly generated parameters for simulated genesis: +{ + stake_per_account: "714557167989", + initially_bonded_validators: "130" +} +Selected randomly generated auth parameters: +{ + "max_memo_characters": "176", + "tx_sig_limit": "6", + "tx_size_cost_per_byte": "14", + "sig_verify_cost_ed25519": "754", + "sig_verify_cost_secp256k1": "719" +} +Selected randomly generated bank parameters: +{ + "send_enabled": true +} +Generated supply parameters: +{ + "supply": [ + { + "denom": "stake", + "amount": "185784863677140" + } + ] +} +Selected randomly generated governance parameters: +{ + "starting_proposal_id": "31", + "deposits": null, + "votes": null, + "proposals": null, + "deposit_params": { + "min_deposit": [ + { + "denom": "stake", + "amount": "496" + } + ], + "max_deposit_period": "11146000000000" + }, + "voting_params": { + "voting_period": "11146000000000" + }, + "tally_params": { + "quorum": "0.348000000000000000", + "threshold": "0.495000000000000000", + "veto": "0.288000000000000000" + } +} +Selected randomly generated minting parameters: +{ + "mint_denom": "stake", + "inflation_rate_change": "0.560000000000000000", + "inflation_max": "0.200000000000000000", + "inflation_min": "0.070000000000000000", + "goal_bonded": "0.670000000000000000", + "blocks_per_year": "6311520" +} +Selected randomly generated distribution parameters: +{ + "fee_pool": { + "community_pool": [] + }, + "community_tax": "0.090000000000000000", + "base_proposer_reward": "0.190000000000000000", + "bonus_proposer_reward": "0.100000000000000000", + "withdraw_addr_enabled": false, + "delegator_withdraw_infos": null, + "previous_proposer": "", + "outstanding_rewards": null, + "validator_accumulated_commissions": null, + "validator_historical_rewards": null, + "validator_current_rewards": null, + "delegator_starting_infos": null, + "validator_slash_events": null +} +Selected randomly generated staking parameters: +{ + "unbonding_time": "163286000000000", + "max_validators": 175, + "max_entries": 7, + "bond_denom": "stake" +} +Selected randomly generated slashing parameters: +{ + "max_evidence_age": "163286000000000", + "signed_blocks_window": "695", + "min_signed_per_window": "0.500000000000000000", + "downtime_jail_duration": "14486000000000", + "slash_fraction_double_sign": "0.024390243902439024", + "slash_fraction_downtime": "0.008130081300813008" +} +Starting the simulation from time Wed Aug 20 16:59:49 UTC 2566 (unixtime 18828003589) + +Simulating... block 8063/10000, operation 0/349. + +Simulation stopped early as all validators have been unbonded; nobody left to propose a block! +Event statistics: + /no-operation/failure => 39403 + auth/deduct_fee/failure => 232 + auth/deduct_fee/ok => 3501 + bank/multisend/failure => 388 + bank/multisend/ok => 5802 + bank/send/failure => 3773 + bank/send/ok => 59815 + beginblock/evidence => 4927 + beginblock/signing/missed => 2741 + beginblock/signing/signed => 19978 + distribution/set_withdraw_address/failure => 31822 + distribution/withdraw_delegator_reward/failure => 8225 + distribution/withdraw_delegator_reward/ok => 23820 + distribution/withdraw_validator_commission/failure => 30578 + distribution/withdraw_validator_commission/ok => 1213 + endblock/validatorupdates/added => 35 + endblock/validatorupdates/kicked => 165 + endblock/validatorupdates/updated => 3909 + gov/deposit/failure => 63464 + gov/deposit/ok => 341 + gov/submit_proposal/failure => 2002 + gov/submit_proposal/ok => 7442 + slashing/unjail/failure => 63077 + slashing/unjail/ok => 29 + staking/begin_redelegate/failure => 32582 + staking/begin_redelegate/ok => 21364 + staking/begin_unbonding/failure => 5 + staking/begin_unbonding/ok => 53397 + staking/create_validator/failure => 53883 + staking/create_validator/ok => 6 + staking/delegate/ok => 53694 + staking/edit_validator/failure => 380 + staking/edit_validator/ok => 2882 +GoLevelDB Stats +Compactions + Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +-------+------------+---------------+---------------+---------------+--------------- + 0 | 3 | 8.20059 | 76.03062 | 0.00000 | 6397.45456 + 1 | 60 | 99.32366 | 969.71138 | 65637.01816 | 59966.39793 + 2 | 273 | 541.33349 | 34.51034 | 2668.15302 | 2590.17643 +-------+------------+---------------+---------------+---------------+--------------- + Total | 336 | 648.85774 | 1080.25234 | 68305.17119 | 68954.02892 + +GoLevelDB cached block size 0 +goos: linux +goarch: amd64 +pkg: github.com/cosmos/cosmos-sdk/simapp +BenchmarkFullAppSimulation-4 1 2880812940613 ns/op 458329227160 B/op 7859938088 allocs/op +PASS +ok github.com/cosmos/cosmos-sdk/simapp 2881.646s diff --git a/benchmarks/results/158-pruning/sdk-pruning-100-2-results.txt b/benchmarks/results/158-pruning/sdk-pruning-100-2-results.txt new file mode 100644 index 000000000..a0c05690e --- /dev/null +++ b/benchmarks/results/158-pruning/sdk-pruning-100-2-results.txt @@ -0,0 +1,163 @@ +Starting SimulateFromSeed with randomness created with seed 53 +Randomized simulation params: +{ + "PastEvidenceFraction": 0.5380706684387375, + "NumKeys": 130, + "EvidenceFraction": 0.38121722232625455, + "InitialLivenessWeightings": [ + 45, + 1, + 6 + ], + "LivenessTransitionMatrix": {}, + "BlockSizeTransitionMatrix": {} +} +Selected randomly generated parameters for simulated genesis: +{ + stake_per_account: "714557167989", + initially_bonded_validators: "130" +} +Selected randomly generated auth parameters: +{ + "max_memo_characters": "176", + "tx_sig_limit": "6", + "tx_size_cost_per_byte": "14", + "sig_verify_cost_ed25519": "754", + "sig_verify_cost_secp256k1": "719" +} +Selected randomly generated bank parameters: +{ + "send_enabled": true +} +Generated supply parameters: +{ + "supply": [ + { + "denom": "stake", + "amount": "185784863677140" + } + ] +} +Selected randomly generated governance parameters: +{ + "starting_proposal_id": "31", + "deposits": null, + "votes": null, + "proposals": null, + "deposit_params": { + "min_deposit": [ + { + "denom": "stake", + "amount": "496" + } + ], + "max_deposit_period": "11146000000000" + }, + "voting_params": { + "voting_period": "11146000000000" + }, + "tally_params": { + "quorum": "0.348000000000000000", + "threshold": "0.495000000000000000", + "veto": "0.288000000000000000" + } +} +Selected randomly generated minting parameters: +{ + "mint_denom": "stake", + "inflation_rate_change": "0.560000000000000000", + "inflation_max": "0.200000000000000000", + "inflation_min": "0.070000000000000000", + "goal_bonded": "0.670000000000000000", + "blocks_per_year": "6311520" +} +Selected randomly generated distribution parameters: +{ + "fee_pool": { + "community_pool": [] + }, + "community_tax": "0.090000000000000000", + "base_proposer_reward": "0.190000000000000000", + "bonus_proposer_reward": "0.100000000000000000", + "withdraw_addr_enabled": false, + "delegator_withdraw_infos": null, + "previous_proposer": "", + "outstanding_rewards": null, + "validator_accumulated_commissions": null, + "validator_historical_rewards": null, + "validator_current_rewards": null, + "delegator_starting_infos": null, + "validator_slash_events": null +} +Selected randomly generated staking parameters: +{ + "unbonding_time": "163286000000000", + "max_validators": 175, + "max_entries": 7, + "bond_denom": "stake" +} +Selected randomly generated slashing parameters: +{ + "max_evidence_age": "163286000000000", + "signed_blocks_window": "695", + "min_signed_per_window": "0.500000000000000000", + "downtime_jail_duration": "14486000000000", + "slash_fraction_double_sign": "0.024390243902439024", + "slash_fraction_downtime": "0.008130081300813008" +} +Starting the simulation from time Wed Aug 20 16:59:49 UTC 2566 (unixtime 18828003589) + +Simulating... block 8063/10000, operation 0/349. + +Simulation stopped early as all validators have been unbonded; nobody left to propose a block! +Event statistics: + /no-operation/failure => 39403 + auth/deduct_fee/failure => 232 + auth/deduct_fee/ok => 3501 + bank/multisend/failure => 388 + bank/multisend/ok => 5802 + bank/send/failure => 3773 + bank/send/ok => 59815 + beginblock/evidence => 4927 + beginblock/signing/missed => 2741 + beginblock/signing/signed => 19978 + distribution/set_withdraw_address/failure => 31822 + distribution/withdraw_delegator_reward/failure => 8225 + distribution/withdraw_delegator_reward/ok => 23820 + distribution/withdraw_validator_commission/failure => 30578 + distribution/withdraw_validator_commission/ok => 1213 + endblock/validatorupdates/added => 35 + endblock/validatorupdates/kicked => 165 + endblock/validatorupdates/updated => 3909 + gov/deposit/failure => 63464 + gov/deposit/ok => 341 + gov/submit_proposal/failure => 2002 + gov/submit_proposal/ok => 7442 + slashing/unjail/failure => 63077 + slashing/unjail/ok => 29 + staking/begin_redelegate/failure => 32582 + staking/begin_redelegate/ok => 21364 + staking/begin_unbonding/failure => 5 + staking/begin_unbonding/ok => 53397 + staking/create_validator/failure => 53883 + staking/create_validator/ok => 6 + staking/delegate/ok => 53694 + staking/edit_validator/failure => 380 + staking/edit_validator/ok => 2882 +GoLevelDB Stats +Compactions + Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +-------+------------+---------------+---------------+---------------+--------------- + 0 | 0 | 0.00000 | 10.04960 | 0.00000 | 766.38953 + 1 | 54 | 98.12634 | 136.22127 | 8386.60535 | 7919.98135 + 2 | 97 | 186.67873 | 4.74288 | 333.74171 | 318.78126 +-------+------------+---------------+---------------+---------------+--------------- + Total | 151 | 284.80508 | 151.01376 | 8720.34706 | 9005.15214 + +GoLevelDB cached block size 0 +goos: linux +goarch: amd64 +pkg: github.com/cosmos/cosmos-sdk/simapp +BenchmarkFullAppSimulation-4 1 2705549310442 ns/op 325911847648 B/op 6971455548 allocs/op +PASS +ok github.com/cosmos/cosmos-sdk/simapp 2706.410s diff --git a/benchmarks/results/158-pruning/sdk-pruning-1000-5-results.txt b/benchmarks/results/158-pruning/sdk-pruning-1000-5-results.txt new file mode 100644 index 000000000..2d069d16f --- /dev/null +++ b/benchmarks/results/158-pruning/sdk-pruning-1000-5-results.txt @@ -0,0 +1,162 @@ +Starting SimulateFromSeed with randomness created with seed 53 +Randomized simulation params: +{ + "PastEvidenceFraction": 0.5380706684387375, + "NumKeys": 130, + "EvidenceFraction": 0.38121722232625455, + "InitialLivenessWeightings": [ + 45, + 1, + 6 + ], + "LivenessTransitionMatrix": {}, + "BlockSizeTransitionMatrix": {} +} +Selected randomly generated parameters for simulated genesis: +{ + stake_per_account: "714557167989", + initially_bonded_validators: "130" +} +Selected randomly generated auth parameters: +{ + "max_memo_characters": "176", + "tx_sig_limit": "6", + "tx_size_cost_per_byte": "14", + "sig_verify_cost_ed25519": "754", + "sig_verify_cost_secp256k1": "719" +} +Selected randomly generated bank parameters: +{ + "send_enabled": true +} +Generated supply parameters: +{ + "supply": [ + { + "denom": "stake", + "amount": "185784863677140" + } + ] +} +Selected randomly generated governance parameters: +{ + "starting_proposal_id": "31", + "deposits": null, + "votes": null, + "proposals": null, + "deposit_params": { + "min_deposit": [ + { + "denom": "stake", + "amount": "496" + } + ], + "max_deposit_period": "11146000000000" + }, + "voting_params": { + "voting_period": "11146000000000" + }, + "tally_params": { + "quorum": "0.348000000000000000", + "threshold": "0.495000000000000000", + "veto": "0.288000000000000000" + } +} +Selected randomly generated minting parameters: +{ + "mint_denom": "stake", + "inflation_rate_change": "0.560000000000000000", + "inflation_max": "0.200000000000000000", + "inflation_min": "0.070000000000000000", + "goal_bonded": "0.670000000000000000", + "blocks_per_year": "6311520" +} +Selected randomly generated distribution parameters: +{ + "fee_pool": { + "community_pool": [] + }, + "community_tax": "0.090000000000000000", + "base_proposer_reward": "0.190000000000000000", + "bonus_proposer_reward": "0.100000000000000000", + "withdraw_addr_enabled": false, + "delegator_withdraw_infos": null, + "previous_proposer": "", + "outstanding_rewards": null, + "validator_accumulated_commissions": null, + "validator_historical_rewards": null, + "validator_current_rewards": null, + "delegator_starting_infos": null, + "validator_slash_events": null +} +Selected randomly generated staking parameters: +{ + "unbonding_time": "163286000000000", + "max_validators": 175, + "max_entries": 7, + "bond_denom": "stake" +} +Selected randomly generated slashing parameters: +{ + "max_evidence_age": "163286000000000", + "signed_blocks_window": "695", + "min_signed_per_window": "0.500000000000000000", + "downtime_jail_duration": "14486000000000", + "slash_fraction_double_sign": "0.024390243902439024", + "slash_fraction_downtime": "0.008130081300813008" +} +Starting the simulation from time Wed Aug 20 16:59:49 UTC 2566 (unixtime 18828003589) + +Simulating... block 8063/10000, operation 0/349. + +Simulation stopped early as all validators have been unbonded; nobody left to propose a block! +Event statistics: + /no-operation/failure => 39403 + auth/deduct_fee/failure => 232 + auth/deduct_fee/ok => 3501 + bank/multisend/failure => 388 + bank/multisend/ok => 5802 + bank/send/failure => 3773 + bank/send/ok => 59815 + beginblock/evidence => 4927 + beginblock/signing/missed => 2741 + beginblock/signing/signed => 19978 + distribution/set_withdraw_address/failure => 31822 + distribution/withdraw_delegator_reward/failure => 8225 + distribution/withdraw_delegator_reward/ok => 23820 + distribution/withdraw_validator_commission/failure => 30578 + distribution/withdraw_validator_commission/ok => 1213 + endblock/validatorupdates/added => 35 + endblock/validatorupdates/kicked => 165 + endblock/validatorupdates/updated => 3909 + gov/deposit/failure => 63464 + gov/deposit/ok => 341 + gov/submit_proposal/failure => 2002 + gov/submit_proposal/ok => 7442 + slashing/unjail/failure => 63077 + slashing/unjail/ok => 29 + staking/begin_redelegate/failure => 32582 + staking/begin_redelegate/ok => 21364 + staking/begin_unbonding/failure => 5 + staking/begin_unbonding/ok => 53397 + staking/create_validator/failure => 53883 + staking/create_validator/ok => 6 + staking/delegate/ok => 53694 + staking/edit_validator/failure => 380 + staking/edit_validator/ok => 2882 +GoLevelDB Stats +Compactions + Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +-------+------------+---------------+---------------+---------------+--------------- + 0 | 1 | 1.77039 | 1.58206 | 0.00000 | 108.22015 + 1 | 47 | 93.77884 | 12.13807 | 666.67667 | 654.00576 +-------+------------+---------------+---------------+---------------+--------------- + Total | 48 | 95.54923 | 13.72012 | 666.67667 | 762.22590 + +GoLevelDB cached block size 0 +goos: linux +goarch: amd64 +pkg: github.com/cosmos/cosmos-sdk/simapp +BenchmarkFullAppSimulation-4 1 2657169989384 ns/op 315683118736 B/op 6960674109 allocs/op +PASS +ok github.com/cosmos/cosmos-sdk/simapp 2658.013s diff --git a/benchmarks/results/158-pruning/sdk-pruning-5-2-results.txt b/benchmarks/results/158-pruning/sdk-pruning-5-2-results.txt new file mode 100644 index 000000000..de8572fdf --- /dev/null +++ b/benchmarks/results/158-pruning/sdk-pruning-5-2-results.txt @@ -0,0 +1,163 @@ +Starting SimulateFromSeed with randomness created with seed 53 +Randomized simulation params: +{ + "PastEvidenceFraction": 0.5380706684387375, + "NumKeys": 130, + "EvidenceFraction": 0.38121722232625455, + "InitialLivenessWeightings": [ + 45, + 1, + 6 + ], + "LivenessTransitionMatrix": {}, + "BlockSizeTransitionMatrix": {} +} +Selected randomly generated parameters for simulated genesis: +{ + stake_per_account: "714557167989", + initially_bonded_validators: "130" +} +Selected randomly generated auth parameters: +{ + "max_memo_characters": "176", + "tx_sig_limit": "6", + "tx_size_cost_per_byte": "14", + "sig_verify_cost_ed25519": "754", + "sig_verify_cost_secp256k1": "719" +} +Selected randomly generated bank parameters: +{ + "send_enabled": true +} +Generated supply parameters: +{ + "supply": [ + { + "denom": "stake", + "amount": "185784863677140" + } + ] +} +Selected randomly generated governance parameters: +{ + "starting_proposal_id": "31", + "deposits": null, + "votes": null, + "proposals": null, + "deposit_params": { + "min_deposit": [ + { + "denom": "stake", + "amount": "496" + } + ], + "max_deposit_period": "11146000000000" + }, + "voting_params": { + "voting_period": "11146000000000" + }, + "tally_params": { + "quorum": "0.348000000000000000", + "threshold": "0.495000000000000000", + "veto": "0.288000000000000000" + } +} +Selected randomly generated minting parameters: +{ + "mint_denom": "stake", + "inflation_rate_change": "0.560000000000000000", + "inflation_max": "0.200000000000000000", + "inflation_min": "0.070000000000000000", + "goal_bonded": "0.670000000000000000", + "blocks_per_year": "6311520" +} +Selected randomly generated distribution parameters: +{ + "fee_pool": { + "community_pool": [] + }, + "community_tax": "0.090000000000000000", + "base_proposer_reward": "0.190000000000000000", + "bonus_proposer_reward": "0.100000000000000000", + "withdraw_addr_enabled": false, + "delegator_withdraw_infos": null, + "previous_proposer": "", + "outstanding_rewards": null, + "validator_accumulated_commissions": null, + "validator_historical_rewards": null, + "validator_current_rewards": null, + "delegator_starting_infos": null, + "validator_slash_events": null +} +Selected randomly generated staking parameters: +{ + "unbonding_time": "163286000000000", + "max_validators": 175, + "max_entries": 7, + "bond_denom": "stake" +} +Selected randomly generated slashing parameters: +{ + "max_evidence_age": "163286000000000", + "signed_blocks_window": "695", + "min_signed_per_window": "0.500000000000000000", + "downtime_jail_duration": "14486000000000", + "slash_fraction_double_sign": "0.024390243902439024", + "slash_fraction_downtime": "0.008130081300813008" +} +Starting the simulation from time Wed Aug 20 16:59:49 UTC 2566 (unixtime 18828003589) + +Simulating... block 8063/10000, operation 0/349. + +Simulation stopped early as all validators have been unbonded; nobody left to propose a block! +Event statistics: + /no-operation/failure => 39403 + auth/deduct_fee/failure => 232 + auth/deduct_fee/ok => 3501 + bank/multisend/failure => 388 + bank/multisend/ok => 5802 + bank/send/failure => 3773 + bank/send/ok => 59815 + beginblock/evidence => 4927 + beginblock/signing/missed => 2741 + beginblock/signing/signed => 19978 + distribution/set_withdraw_address/failure => 31822 + distribution/withdraw_delegator_reward/failure => 8225 + distribution/withdraw_delegator_reward/ok => 23820 + distribution/withdraw_validator_commission/failure => 30578 + distribution/withdraw_validator_commission/ok => 1213 + endblock/validatorupdates/added => 35 + endblock/validatorupdates/kicked => 165 + endblock/validatorupdates/updated => 3909 + gov/deposit/failure => 63464 + gov/deposit/ok => 341 + gov/submit_proposal/failure => 2002 + gov/submit_proposal/ok => 7442 + slashing/unjail/failure => 63077 + slashing/unjail/ok => 29 + staking/begin_redelegate/failure => 32582 + staking/begin_redelegate/ok => 21364 + staking/begin_unbonding/failure => 5 + staking/begin_unbonding/ok => 53397 + staking/create_validator/failure => 53883 + staking/create_validator/ok => 6 + staking/delegate/ok => 53694 + staking/edit_validator/failure => 380 + staking/edit_validator/ok => 2882 +GoLevelDB Stats +Compactions + Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +-------+------------+---------------+---------------+---------------+--------------- + 0 | 3 | 8.20775 | 157.18686 | 0.00000 | 12515.15473 + 1 | 65 | 99.16383 | 1962.98229 | 129484.30484 | 117927.78594 + 2 | 368 | 733.14210 | 58.03251 | 4574.45495 | 4456.33280 +-------+------------+---------------+---------------+---------------+--------------- + Total | 436 | 840.51368 | 2178.20165 | 134058.75979 | 134899.27347 + +GoLevelDB cached block size 0 +goos: linux +goarch: amd64 +pkg: github.com/cosmos/cosmos-sdk/simapp +BenchmarkFullAppSimulation-4 1 3603346114000 ns/op 606564761648 B/op 8823765100 allocs/op +PASS +ok github.com/cosmos/cosmos-sdk/simapp 3604.335s From f661c7015b193a974717fc5d964e5e9fd514cd08 Mon Sep 17 00:00:00 2001 From: Aditya Sripal Date: Mon, 7 Oct 2019 17:39:25 -0700 Subject: [PATCH 46/57] fix tiny bug --- mutable_tree_test.go | 19 +++++++++++++++++++ nodedb.go | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/mutable_tree_test.go b/mutable_tree_test.go index fe41f71a8..77c77790b 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -43,3 +43,22 @@ func TestTraverse(t *testing.T) { require.Equal(t, 11, tree.nodeSize(), "Size of tree unexpected") } + +func TestEmptyRecents(t *testing.T) { + memDB := db.NewMemDB() + opts := Options{ + KeepRecent: 100, + KeepEvery: 10000, + } + + tree := NewMutableTreeWithOpts(memDB, db.NewMemDB(), 0, &opts) + hash, version, err := tree.SaveVersion() + + require.Nil(t, err) + require.Equal(t, int64(1), version) + require.Nil(t, hash) + require.True(t, tree.VersionExists(int64(1))) + + _, err = tree.GetImmutable(int64(1)) + require.Nil(t, err) +} diff --git a/nodedb.go b/nodedb.go index 050166aa6..ba1be2a60 100644 --- a/nodedb.go +++ b/nodedb.go @@ -528,7 +528,7 @@ func (ndb *nodeDB) getRoot(version int64) []byte { if ndb.isRecentVersion(version) { memroot := ndb.recentDB.Get(ndb.rootKey(version)) // TODO: maybe I shouldn't check in snapshot if it isn't here - if len(memroot) > 0 { + if memroot != nil { return memroot } } From 1b742539cc482c181e6404486a79258c570a9a44 Mon Sep 17 00:00:00 2001 From: Marko Baricevic Date: Wed, 4 Dec 2019 13:04:52 +0100 Subject: [PATCH 47/57] minor linting fixes --- .golangci.yml | 6 ++++++ benchmarks/prune_test.go | 1 + cmd/iaviewer/main.go | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a2ea7bbfe..97a791daf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -21,3 +21,9 @@ linters: - gochecknoglobals - gocritic - gochecknoinits + - godox + - wsl + - whitespace + - funlen + - gocognit + - errcheck diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index 1f3f17e01..f2b1dcf62 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -89,6 +89,7 @@ func BenchmarkPruningStrategies(b *testing.B) { // {10000, 100}, // SDK pruning } for _, ps := range ps { + ps := ps prefix := fmt.Sprintf("PruningStrategy{%d-%d}-KeyLen:%d-DataLen:%d", ps.keepEvery, ps.keepRecent, 16, 40) b.Run(prefix, func(sub *testing.B) { diff --git a/cmd/iaviewer/main.go b/cmd/iaviewer/main.go index 92393507a..ffa4df304 100644 --- a/cmd/iaviewer/main.go +++ b/cmd/iaviewer/main.go @@ -117,15 +117,15 @@ func PrintKeys(tree *iavl.MutableTree) { func parseWeaveKey(key []byte) string { cut := bytes.IndexRune(key, ':') if cut == -1 { - return encodeId(key) + return encodeID(key) } prefix := key[:cut] id := key[cut+1:] - return fmt.Sprintf("%s:%s", encodeId(prefix), encodeId(id)) + return fmt.Sprintf("%s:%s", encodeID(prefix), encodeID(id)) } // casts to a string if it is printable ascii, hex-encodes otherwise -func encodeId(id []byte) string { +func encodeID(id []byte) string { for _, b := range id { if b < 0x20 || b >= 0x80 { return strings.ToUpper(hex.EncodeToString(id)) From fbdde5d34d03b53725f3934cadfd37f17f7f7982 Mon Sep 17 00:00:00 2001 From: Marko Baricevic Date: Fri, 6 Dec 2019 13:05:04 +0100 Subject: [PATCH 48/57] fix linting issues --- logger.go | 8 -------- nodedb.go | 18 +++++++++--------- pruning_test.go | 19 ++++++++++++------- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/logger.go b/logger.go index 0798fa058..6a429a823 100644 --- a/logger.go +++ b/logger.go @@ -13,11 +13,3 @@ func debug(format string, args ...interface{}) { fmt.Printf(format, args...) } } - -func startDebugger() { - debugging = true -} - -func endDebugger() { - debugging = false -} diff --git a/nodedb.go b/nodedb.go index f6615f657..56922f059 100644 --- a/nodedb.go +++ b/nodedb.go @@ -411,15 +411,15 @@ func traverseOrphansFromDB(db dbm.DB, fn func(k, v []byte)) { // Traverse orphans ending at a certain version. // NOTE: If orphan is in recentDB and levelDB (version > latestVersion-keepRecent && version%keepEvery == 0) // traverse will return the node twice. -func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { - prefix := orphanKeyFormat.Key(version) - if ndb.isRecentVersion(version) { - traversePrefixFromDB(ndb.recentDB, prefix, fn) - } - if ndb.isSnapshotVersion(version) { - traversePrefixFromDB(ndb.snapshotDB, prefix, fn) - } -} +// func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { +// prefix := orphanKeyFormat.Key(version) +// if ndb.isRecentVersion(version) { +// traversePrefixFromDB(ndb.recentDB, prefix, fn) +// } +// if ndb.isSnapshotVersion(version) { +// traversePrefixFromDB(ndb.snapshotDB, prefix, fn) +// } +// } func traverseOrphansVersionFromDB(db dbm.DB, version int64, fn func(k, v []byte)) { prefix := orphanKeyFormat.Key(version) diff --git a/pruning_test.go b/pruning_test.go index f306c7853..807a6c825 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -60,10 +60,10 @@ func TestSave(t *testing.T) { // check all expected versions are available. for j := keepEvery; j <= mt.Version(); j += keepEvery { - require.True(t, mt.VersionExists(int64(j)), "Expected snapshot version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", j, PruningOptions(keepEvery, keepRecent)) + require.True(t, mt.VersionExists(j), "Expected snapshot version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", j, PruningOptions(keepEvery, keepRecent)) } for k := mt.Version() - keepRecent + 1; k <= mt.Version(); k++ { - require.True(t, mt.VersionExists(int64(k)), "Expected recent version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", k, PruningOptions(keepEvery, keepRecent)) + require.True(t, mt.VersionExists(k), "Expected recent version: %d to be available in nodeDB. KeepEvery: %d, KeepRecent: %d", k, PruningOptions(keepEvery, keepRecent)) } // check that there only exists correct number of roots in nodeDB @@ -201,7 +201,8 @@ func TestRemoveKeys(t *testing.T) { for i := 0; i < 10; i++ { mt.Set([]byte(fmt.Sprintf("v%d:%d", v, i)), []byte(fmt.Sprintf("Val:v:%d:%d", v, i))) } - mt.SaveVersion() + _, _, err := mt.SaveVersion() + require.NoError(t, err) } numNodes := mt.nodeSize() @@ -212,13 +213,15 @@ func TestRemoveKeys(t *testing.T) { _, removed := mt.Remove([]byte(key)) require.True(t, removed, "Key %s could not be removed", key) } - mt.SaveVersion() + _, _, err := mt.SaveVersion() + require.NoError(t, err) } require.Equal(t, numNodes, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "Number of Nodes in snapshotDB are unexpected") // Delete only non-empty tree in snapshotDB - mt.DeleteVersion(10) + err := mt.DeleteVersion(10) + require.NoError(t, err) require.Equal(t, 0, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "Still have nodes in snapshotDB") } @@ -335,7 +338,8 @@ func TestSanity2(t *testing.T) { require.Equal(t, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "DB sizes should be the same") for i := 1; i < 5; i++ { - mt.DeleteVersion(int64(i)) + err := mt.DeleteVersion(int64(i)) + require.NoError(t, err) } require.Equal(t, mt.nodeSize()+size, len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) @@ -377,7 +381,8 @@ func TestSanity3(t *testing.T) { require.Equal(t, numSnapNodes, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) - mt.DeleteVersion(100) + err := mt.DeleteVersion(100) + require.NoError(t, err) require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) } From 126ac73f6797e77cc6839d6a4640343786c71759 Mon Sep 17 00:00:00 2001 From: Marko Date: Fri, 6 Dec 2019 13:07:59 +0100 Subject: [PATCH 49/57] Update PRUNING.md Co-Authored-By: Ismail Khoffi --- PRUNING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PRUNING.md b/PRUNING.md index 0fa5e2855..346a18980 100644 --- a/PRUNING.md +++ b/PRUNING.md @@ -2,7 +2,7 @@ Setting Pruning fields in the IAVL tree can optimize performance by only writing versions to disk if they are meant to be persisted indefinitely. Versions that are known to be deleted eventually are temporarily held in memory until they are ready to be pruned. This greatly reduces the I/O load of IAVL. -We can set custom pruning fields in IAVL using: `NewMutableTreePruningOpts` +We can set custom pruning fields in IAVL using: `NewMutableTreeWithOpts` ## Current design From 59317ba084d14cb31d90cac97daa0bc36f0dcca7 Mon Sep 17 00:00:00 2001 From: Marko Baricevic Date: Fri, 6 Dec 2019 13:25:14 +0100 Subject: [PATCH 50/57] fix test --- pruning_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pruning_test.go b/pruning_test.go index 807a6c825..f74d922cd 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -339,7 +339,7 @@ func TestSanity2(t *testing.T) { for i := 1; i < 5; i++ { err := mt.DeleteVersion(int64(i)) - require.NoError(t, err) + require.NotNil(t, err) } require.Equal(t, mt.nodeSize()+size, len(mt.ndb.nodesFromDB(mt.ndb.recentDB))) From 84deb831e042af52706a11af918ac200f723728a Mon Sep 17 00:00:00 2001 From: Timothy Chen Date: Wed, 18 Dec 2019 22:06:49 -0800 Subject: [PATCH 51/57] Add options validation --- basic_test.go | 24 ++++--- mutable_tree.go | 26 +++++++- mutable_tree_test.go | 12 ++-- proof_test.go | 9 ++- pruning_test.go | 47 ++++++++++---- testutils_test.go | 9 ++- tree_dotgraph_test.go | 5 +- tree_fuzz_test.go | 6 +- tree_test.go | 146 +++++++++++++++++++++++++++--------------- 9 files changed, 198 insertions(+), 86 deletions(-) diff --git a/basic_test.go b/basic_test.go index e3a39ef0e..de7d09fd6 100644 --- a/basic_test.go +++ b/basic_test.go @@ -13,7 +13,8 @@ import ( ) func TestBasic(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) up := tree.Set([]byte("1"), []byte("one")) if up { t.Error("Did not expect an update (should have been create)") @@ -189,7 +190,8 @@ func TestRemove(t *testing.T) { keyLen, dataLen := 16, 40 size := 10000 - t1 := getTestTree(size) + t1, err := getTestTree(size) + require.NoError(t, err) // insert a bunch of random nodes keys := make([][]byte, size) @@ -219,7 +221,8 @@ func TestIntegration(t *testing.T) { } records := make([]*record, 400) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) randomRecord := func() *record { return &record{randstr(20), randstr(20)} @@ -301,7 +304,8 @@ func TestIterateRange(t *testing.T) { } sort.Strings(keys) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) // insert all the data for _, r := range records { @@ -371,14 +375,16 @@ func TestPersistence(t *testing.T) { } // Construct some tree and save it - t1 := NewMutableTree(db, 0) + t1, err := NewMutableTree(db, 0) + require.NoError(t, err) for key, value := range records { t1.Set([]byte(key), []byte(value)) } t1.SaveVersion() // Load a tree - t2 := NewMutableTree(db, 0) + t2, err := NewMutableTree(db, 0) + require.NoError(t, err) t2.Load() for key, value := range records { _, t2value := t2.Get([]byte(key)) @@ -391,7 +397,8 @@ func TestPersistence(t *testing.T) { func TestProof(t *testing.T) { // Construct some random tree - tree := getTestTree(100) + tree, err := getTestTree(100) + require.NoError(t, err) for i := 0; i < 10; i++ { key, value := randstr(20), randstr(20) tree.Set([]byte(key), []byte(value)) @@ -420,7 +427,8 @@ func TestProof(t *testing.T) { func TestTreeProof(t *testing.T) { db := db.NewMemDB() - tree := NewMutableTree(db, 100) + tree, err := NewMutableTree(db, 100) + require.NoError(t, err) assert.Equal(t, tree.Hash(), []byte(nil)) // should get false for proof with nil root diff --git a/mutable_tree.go b/mutable_tree.go index d5edbe605..5b80aef23 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -24,14 +24,34 @@ type MutableTree struct { // NewMutableTree returns a new tree with the specified cache size and datastore // To maintain backwards compatibility, this function will initialize PruningStrategy{keepEvery: 1, keepRecent: 0} -func NewMutableTree(db dbm.DB, cacheSize int) *MutableTree { +func NewMutableTree(db dbm.DB, cacheSize int) (*MutableTree, error) { // memDB is initialized but should never be written to memDB := dbm.NewMemDB() return NewMutableTreeWithOpts(db, memDB, cacheSize, nil) } +func validateOptions(opts *Options) error { + switch { + case opts == nil: + return nil + case opts.KeepEvery < 0: + return errors.New("keep every cannot be negative") + case opts.KeepRecent < 0: + return errors.New("keep recent cannot be negative") + case opts.KeepRecent == 0 && opts.KeepEvery > 1: + // We cannot snapshot more than every one version when we don't keep any versions in memory. + return errors.New("keep recent cannot be zero when keep every is set larger than one") + } + + return nil +} + // NewMutableTreeWithOpts returns a new tree with the specified cache size, datastores and options -func NewMutableTreeWithOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, opts *Options) *MutableTree { +func NewMutableTreeWithOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, opts *Options) (*MutableTree, error) { + if err := validateOptions(opts); err != nil { + return nil, err + } + ndb := newNodeDB(snapDB, recentDB, cacheSize, opts) head := &ImmutableTree{ndb: ndb} @@ -41,7 +61,7 @@ func NewMutableTreeWithOpts(snapDB dbm.DB, recentDB dbm.DB, cacheSize int, opts orphans: map[string]int64{}, versions: map[int64]bool{}, ndb: ndb, - } + }, nil } // IsEmpty returns whether or not the tree has any keys. Only trees that are diff --git a/mutable_tree_test.go b/mutable_tree_test.go index d74ea99c1..1cad402ad 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -12,7 +12,8 @@ import ( func TestDelete(t *testing.T) { memDb := db.NewMemDB() - tree := NewMutableTree(memDb, 0) + tree, err := NewMutableTree(memDb, 0) + require.NoError(t, err) tree.set([]byte("k1"), []byte("Fred")) hash, version, err := tree.SaveVersion() @@ -36,7 +37,8 @@ func TestDelete(t *testing.T) { func TestTraverse(t *testing.T) { memDb := db.NewMemDB() - tree := NewMutableTree(memDb, 0) + tree, err := NewMutableTree(memDb, 0) + require.NoError(t, err) for i := 0; i < 6; i++ { tree.set([]byte(fmt.Sprintf("k%d", i)), []byte(fmt.Sprintf("v%d", i))) @@ -52,7 +54,8 @@ func TestEmptyRecents(t *testing.T) { KeepEvery: 10000, } - tree := NewMutableTreeWithOpts(memDB, db.NewMemDB(), 0, &opts) + tree, err := NewMutableTreeWithOpts(memDB, db.NewMemDB(), 0, &opts) + require.NoError(t, err) hash, version, err := tree.SaveVersion() require.Nil(t, err) @@ -66,7 +69,8 @@ func TestEmptyRecents(t *testing.T) { func BenchmarkMutableTree_Set(b *testing.B) { db := db.NewDB("test", db.MemDBBackend, "") - t := NewMutableTree(db, 100000) + t, err := NewMutableTree(db, 100000) + require.NoError(b, err) for i := 0; i < 1000000; i++ { t.Set(randBytes(10), []byte{}) } diff --git a/proof_test.go b/proof_test.go index 8b4bcebf1..4184a1903 100644 --- a/proof_test.go +++ b/proof_test.go @@ -13,7 +13,8 @@ import ( ) func TestTreeGetWithProof(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) require := require.New(t) for _, ikey := range []byte{0x11, 0x32, 0x50, 0x72, 0x99} { key := []byte{ikey} @@ -47,7 +48,8 @@ func TestTreeGetWithProof(t *testing.T) { } func TestTreeKeyExistsProof(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) root := tree.WorkingHash() // should get false for proof with nil root @@ -113,7 +115,8 @@ func TestTreeKeyExistsProof(t *testing.T) { } func TestTreeKeyInRangeProofs(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) require := require.New(t) keys := []byte{0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7} // 10 total. for _, ikey := range keys { diff --git a/pruning_test.go b/pruning_test.go index f74d922cd..affe10762 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -28,7 +28,8 @@ func TestSave(t *testing.T) { keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (rand.Int63n(3) + 1) * 100 - mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + require.NoError(t, err) // create 1000 versions for i := 0; i < 1000; i++ { @@ -83,7 +84,8 @@ func TestDeleteOrphans(t *testing.T) { keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (rand.Int63n(3) + 1) * 100 - mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + require.NoError(t, err) // create 1200 versions (multiple of any possible snapshotting version) for i := 0; i < 1200; i++ { @@ -159,7 +161,8 @@ func TestReplaceKeys(t *testing.T) { keepRecent := int64(1) //keep 1 version in memDB keepEvery := int64(5) - mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + require.NoError(t, err) // Replace the same 10 keys with different values for i := 0; i < 10; i++ { @@ -195,7 +198,8 @@ func TestRemoveKeys(t *testing.T) { keepRecent := int64(1) //keep 1 version in memDB keepEvery := int64(10) - mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + require.NoError(t, err) for v := 0; v < 10; v++ { for i := 0; i < 10; i++ { @@ -220,7 +224,7 @@ func TestRemoveKeys(t *testing.T) { require.Equal(t, numNodes, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "Number of Nodes in snapshotDB are unexpected") // Delete only non-empty tree in snapshotDB - err := mt.DeleteVersion(10) + err = mt.DeleteVersion(10) require.NoError(t, err) require.Equal(t, 0, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "Still have nodes in snapshotDB") } @@ -231,7 +235,8 @@ func TestDBState(t *testing.T) { keepRecent := int64(5) keepEvery := int64(1) - mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + require.NoError(t, err) // create 5 versions for i := 0; i < 5; i++ { @@ -264,7 +269,8 @@ func TestSanity1(t *testing.T) { keepRecent := int64(1) keepEvery := int64(5) - mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + require.NoError(t, err) // create 5 versions for i := 0; i < 5; i++ { @@ -310,7 +316,8 @@ func TestSanity2(t *testing.T) { keepRecent := int64(1) keepEvery := int64(5) - mt := NewMutableTreeWithOpts(db, mdb, 0, PruningOptions(keepEvery, keepRecent)) + mt, err := NewMutableTreeWithOpts(db, mdb, 0, PruningOptions(keepEvery, keepRecent)) + require.NoError(t, err) // create 5 versions for i := 0; i < 5; i++ { @@ -352,7 +359,8 @@ func TestSanity3(t *testing.T) { keepRecent := int64(4) keepEvery := int64(100) - mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(keepEvery, keepRecent)) + require.NoError(t, err) // create 1000 versions numSnapNodes := 0 @@ -381,7 +389,7 @@ func TestSanity3(t *testing.T) { require.Equal(t, numSnapNodes, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) - err := mt.DeleteVersion(100) + err = mt.DeleteVersion(100) require.NoError(t, err) require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB))) @@ -393,7 +401,8 @@ func TestNoSnapshots(t *testing.T) { defer close() keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB - mt := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(0, keepRecent)) // test no snapshots + mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(0, keepRecent)) // test no snapshots + require.NoError(t, err) for i := 0; i < 50; i++ { // set 5 keys per version @@ -432,7 +441,8 @@ func TestNoRecents(t *testing.T) { db, _, close := getTestDBs() defer close() - mt := NewMutableTree(db, 5) + mt, err := NewMutableTree(db, 5) + require.NoError(t, err) for i := 0; i < 50; i++ { // set 5 keys per version @@ -466,3 +476,16 @@ func TestNoRecents(t *testing.T) { require.True(t, seen, "Version %d is not available even though it is snpashhot version", i) } } + +func TestValidationOptions(t *testing.T) { + db, mdb, close := getTestDBs() + defer close() + + _, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(2, 0)) + require.Error(t, err) + _, err = NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(-1, 0)) + require.Error(t, err) + _, err = NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(0, -1)) + require.Error(t, err) +} + diff --git a/testutils_test.go b/testutils_test.go index 4ae9d78ee..167443ab6 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -4,6 +4,7 @@ package iavl import ( "bytes" "fmt" + "github.com/stretchr/testify/require" "runtime" "testing" @@ -30,7 +31,7 @@ func b2i(bz []byte) int { } // Construct a MutableTree with random pruning parameters -func getTestTree(cacheSize int) *MutableTree { +func getTestTree(cacheSize int) (*MutableTree, error) { keepRecent := mrand.Int63n(8) + 2 //keep at least 2 versions in memDB keepEvery := (mrand.Int63n(3) + 1) * 100 // snapshot every {100,200,300} versions @@ -66,7 +67,7 @@ func N(l, r interface{}) *Node { // Setup a deep node func T(n *Node) *MutableTree { - t := getTestTree(0) + t, _ := getTestTree(0) n.hashWithCount() t.root = n @@ -127,7 +128,9 @@ func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { b.StopTimer() - t := NewMutableTree(db, 100000) + t, err := NewMutableTree(db, 100000) + require.NoError(b, err) + value := []byte{} for i := 0; i < 1000000; i++ { t.Set(i2b(int(cmn.RandInt31())), value) diff --git a/tree_dotgraph_test.go b/tree_dotgraph_test.go index 5d6b80047..017bcaebd 100644 --- a/tree_dotgraph_test.go +++ b/tree_dotgraph_test.go @@ -3,10 +3,13 @@ package iavl import ( "io/ioutil" "testing" + + "github.com/stretchr/testify/require" ) func TestWriteDOTGraph(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, } { diff --git a/tree_fuzz_test.go b/tree_fuzz_test.go index d7da5627c..db58bf456 100644 --- a/tree_fuzz_test.go +++ b/tree_fuzz_test.go @@ -3,6 +3,7 @@ package iavl import ( "fmt" + "github.com/stretchr/testify/require" "math/rand" "testing" @@ -111,9 +112,10 @@ func TestMutableTreeFuzz(t *testing.T) { for size := 5; iterations < maxIterations; size++ { for i := 0; i < progsPerIteration/size; i++ { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) program := genRandomProgram(size) - err := program.Execute(tree) + err = program.Execute(tree) if err != nil { t.Fatalf("Error after %d iterations (size %d): %s\n%s", iterations, size, err.Error(), tree.String()) } diff --git a/tree_test.go b/tree_test.go index f1d38a9dc..364cd871b 100644 --- a/tree_test.go +++ b/tree_test.go @@ -48,7 +48,8 @@ func TestVersionedRandomTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewMutableTree(d, 100) + tree, err := NewMutableTree(d, 100) + require.NoError(err) versions := 50 keysPerVersion := 30 @@ -100,8 +101,10 @@ func TestVersionedRandomTreeSmallKeys(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewMutableTree(d, 100) - singleVersionTree := getTestTree(0) + tree, err := NewMutableTree(d, 100) + require.NoError(err) + singleVersionTree, err := getTestTree(0) + require.NoError(err) versions := 20 keysPerVersion := 50 @@ -141,8 +144,10 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewMutableTree(d, 100) - singleVersionTree := getTestTree(0) + tree, err := NewMutableTree(d, 100) + require.NoError(err) + singleVersionTree, err := getTestTree(0) + require.NoError(err) versions := 30 keysPerVersion := 50 @@ -178,7 +183,8 @@ func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { } func TestVersionedTreeSpecial1(t *testing.T) { - tree := getTestTree(100) + tree, err := getTestTree(100) + require.NoError(t, err) tree.Set([]byte("C"), []byte("so43QQFN")) tree.SaveVersion() @@ -201,7 +207,8 @@ func TestVersionedTreeSpecial1(t *testing.T) { func TestVersionedRandomTreeSpecial2(t *testing.T) { require := require.New(t) - tree := getTestTree(100) + tree, err := getTestTree(100) + require.NoError(err) tree.Set([]byte("OFMe2Yvm"), []byte("ez2OtQtE")) tree.Set([]byte("WEN4iN7Y"), []byte("kQNyUalI")) @@ -220,7 +227,8 @@ func TestVersionedEmptyTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0) + require.NoError(err) hash, v, err := tree.SaveVersion() require.Nil(hash) @@ -258,7 +266,8 @@ func TestVersionedEmptyTree(t *testing.T) { // Now reload the tree. - tree = NewMutableTree(d, 0) + tree, err = NewMutableTree(d, 0) + require.NoError(err) tree.Load() require.False(tree.VersionExists(1)) @@ -276,7 +285,8 @@ func TestVersionedTree(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0) + require.NoError(err) // We start with zero keys in the databse. require.Equal(0, tree.ndb.size()) @@ -320,7 +330,8 @@ func TestVersionedTree(t *testing.T) { // Recreate a new tree and load it, to make sure it works in this // scenario. - tree = NewMutableTree(d, 100) + tree, err = NewMutableTree(d, 100) + require.NoError(err) _, err = tree.Load() require.NoError(err) @@ -366,7 +377,8 @@ func TestVersionedTree(t *testing.T) { require.EqualValues(hash3, hash4) require.NotNil(hash4) - tree = NewMutableTree(d, 100) + tree, err = NewMutableTree(d, 100) + require.NoError(err) _, err = tree.Load() require.NoError(err) @@ -461,7 +473,8 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0) + require.NoError(t, err) tree.Set([]byte("key0"), []byte("val0")) tree.Set([]byte("key1"), []byte("val0")) @@ -492,7 +505,8 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { require.Len(t, tree.ndb.leafNodes(), 3) - tree2 := getTestTree(0) + tree2, err := getTestTree(0) + require.NoError(t, err) tree2.Set([]byte("key0"), []byte("val2")) tree2.Set([]byte("key2"), []byte("val2")) tree2.Set([]byte("key3"), []byte("val1")) @@ -502,7 +516,8 @@ func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { } func TestVersionedTreeOrphanDeleting(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) tree.Set([]byte("key0"), []byte("val0")) tree.Set([]byte("key1"), []byte("val0")) @@ -543,7 +558,8 @@ func TestVersionedTreeSpecialCase(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0) + require.NoError(err) tree.Set([]byte("key1"), []byte("val0")) tree.Set([]byte("key2"), []byte("val0")) @@ -566,7 +582,8 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { require := require.New(t) d := db.NewMemDB() - tree := NewMutableTree(d, 100) + tree, err := NewMutableTree(d, 100) + require.NoError(err) tree.Set([]byte("key1"), []byte("val0")) tree.Set([]byte("key2"), []byte("val0")) @@ -579,8 +596,9 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { tree.Set([]byte("key2"), []byte("val2")) tree.SaveVersion() - tree = NewMutableTree(d, 100) - _, err := tree.Load() + tree, err = NewMutableTree(d, 100) + require.NoError(err) + _, err = tree.Load() require.NoError(err) require.NoError(tree.DeleteVersion(2)) @@ -591,7 +609,8 @@ func TestVersionedTreeSpecialCase2(t *testing.T) { func TestVersionedTreeSpecialCase3(t *testing.T) { require := require.New(t) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(err) tree.Set([]byte("m"), []byte("liWT0U6G")) tree.Set([]byte("G"), []byte("7PxRXwUA")) @@ -620,7 +639,8 @@ func TestVersionedTreeSpecialCase3(t *testing.T) { func TestVersionedTreeSaveAndLoad(t *testing.T) { require := require.New(t) d := db.NewMemDB() - tree := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0) + require.NoError(err) // Loading with an empty root is a no-op. tree.Load() @@ -644,7 +664,8 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { require.Equal(int64(6), tree.Version()) // Reload the tree, to test that roots and orphans are properly loaded. - ntree := NewMutableTree(d, 0) + ntree, err := NewMutableTree(d, 0) + require.NoError(err) ntree.Load() require.False(ntree.IsEmpty()) @@ -670,7 +691,8 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { func TestVersionedTreeErrors(t *testing.T) { require := require.New(t) - tree := getTestTree(100) + tree, err := getTestTree(100) + require.NoError(err) // Can't delete non-existent versions. require.Error(tree.DeleteVersion(1)) @@ -679,7 +701,7 @@ func TestVersionedTreeErrors(t *testing.T) { tree.Set([]byte("key"), []byte("val")) // Saving with content is ok. - _, _, err := tree.SaveVersion() + _, _, err = tree.SaveVersion() require.NoError(err) // Can't delete current version. @@ -702,7 +724,8 @@ func TestVersionedCheckpoints(t *testing.T) { d, closeDB := getTestDB() defer closeDB() - tree := NewMutableTree(d, 100) + tree, err := NewMutableTree(d, 100) + require.NoError(err) versions := 50 keysPerVersion := 10 versionsPerCheckpoint := 5 @@ -755,7 +778,8 @@ func TestVersionedCheckpoints(t *testing.T) { func TestVersionedCheckpointsSpecialCase(t *testing.T) { require := require.New(t) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(err) key := []byte("k") tree.Set(key, []byte("val1")) @@ -780,7 +804,8 @@ func TestVersionedCheckpointsSpecialCase(t *testing.T) { } func TestVersionedCheckpointsSpecialCase2(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -800,7 +825,8 @@ func TestVersionedCheckpointsSpecialCase2(t *testing.T) { } func TestVersionedCheckpointsSpecialCase3(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) tree.Set([]byte("n"), []byte("2wUCUs8q")) tree.Set([]byte("l"), []byte("WQ7mvMbc")) @@ -820,7 +846,8 @@ func TestVersionedCheckpointsSpecialCase3(t *testing.T) { } func TestVersionedCheckpointsSpecialCase4(t *testing.T) { - tree := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0) + require.NoError(t, err) tree.Set([]byte("U"), []byte("XamDUtiJ")) tree.Set([]byte("A"), []byte("UkZBuYIU")) @@ -852,7 +879,8 @@ func TestVersionedCheckpointsSpecialCase4(t *testing.T) { } func TestVersionedCheckpointsSpecialCase5(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) tree.Set([]byte("R"), []byte("ygZlIzeW")) tree.SaveVersion() @@ -869,7 +897,8 @@ func TestVersionedCheckpointsSpecialCase5(t *testing.T) { } func TestVersionedCheckpointsSpecialCase6(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) tree.Set([]byte("Y"), []byte("MW79JQeV")) tree.Set([]byte("7"), []byte("Kp0ToUJB")) @@ -901,7 +930,8 @@ func TestVersionedCheckpointsSpecialCase6(t *testing.T) { } func TestVersionedCheckpointsSpecialCase7(t *testing.T) { - tree := getTestTree(100) + tree, err := getTestTree(100) + require.NoError(t, err) tree.Set([]byte("n"), []byte("OtqD3nyn")) tree.Set([]byte("W"), []byte("kMdhJjF5")) @@ -935,7 +965,8 @@ func TestVersionedCheckpointsSpecialCase7(t *testing.T) { func TestVersionedTreeEfficiency(t *testing.T) { require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 0) + tree, err := NewMutableTree(db.NewMemDB(), 0) + require.NoError(err) versions := 20 keysPerVersion := 100 keysAddedPerVersion := map[int]int{} @@ -972,7 +1003,8 @@ func TestVersionedTreeEfficiency(t *testing.T) { func TestVersionedTreeProofs(t *testing.T) { require := require.New(t) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(err) tree.Set([]byte("k1"), []byte("v1")) tree.Set([]byte("k2"), []byte("v1")) @@ -1045,7 +1077,8 @@ func TestOrphans(t *testing.T) { //Then randomly delete versions other than the first and last until only those two remain //Any remaining orphan nodes should either have fromVersion == firstVersion || toVersion == lastVersion require := require.New(t) - tree := NewMutableTree(db.NewMemDB(), 100) + tree, err := NewMutableTree(db.NewMemDB(), 100) + require.NoError(err) NUMVERSIONS := 100 NUMUPDATES := 100 @@ -1074,7 +1107,8 @@ func TestOrphans(t *testing.T) { func TestVersionedTreeHash(t *testing.T) { require := require.New(t) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(err) require.Nil(tree.Hash()) tree.Set([]byte("I"), []byte("D")) @@ -1096,7 +1130,8 @@ func TestVersionedTreeHash(t *testing.T) { func TestNilValueSemantics(t *testing.T) { require := require.New(t) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(err) require.Panics(func() { tree.Set([]byte("k"), nil) @@ -1106,7 +1141,8 @@ func TestNilValueSemantics(t *testing.T) { func TestCopyValueSemantics(t *testing.T) { require := require.New(t) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(err) val := []byte("v1") @@ -1123,7 +1159,8 @@ func TestCopyValueSemantics(t *testing.T) { func TestRollback(t *testing.T) { require := require.New(t) - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(err) tree.Set([]byte("k"), []byte("v")) tree.SaveVersion() @@ -1150,7 +1187,8 @@ func TestRollback(t *testing.T) { } func TestLazyLoadVersion(t *testing.T) { - tree := getTestTree(0) + tree, err := getTestTree(0) + require.NoError(t, err) maxVersions := 10 version, err := tree.LazyLoadVersion(0) @@ -1190,11 +1228,12 @@ func TestOverwrite(t *testing.T) { require := require.New(t) mdb := db.NewMemDB() - tree := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0) + require.NoError(err) // Set one kv pair and save version 1 tree.Set([]byte("key1"), []byte("value1")) - _, _, err := tree.SaveVersion() + _, _, err = tree.SaveVersion() require.NoError(err, "SaveVersion should not fail") // Set another kv pair and save version 2 @@ -1203,7 +1242,8 @@ func TestOverwrite(t *testing.T) { require.NoError(err, "SaveVersion should not fail") // Reload tree at version 1 - tree = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0) + require.NoError(err) _, err = tree.LoadVersion(int64(1)) require.NoError(err, "LoadVersion should not fail") @@ -1222,7 +1262,8 @@ func TestLoadVersionForOverwriting(t *testing.T) { require := require.New(t) mdb := db.NewMemDB() - tree := NewMutableTree(mdb, 0) + tree, err := NewMutableTree(mdb, 0) + require.NoError(err) maxLength := 100 for count := 1; count <= maxLength; count++ { @@ -1233,12 +1274,14 @@ func TestLoadVersionForOverwriting(t *testing.T) { require.NoError(err, "SaveVersion should not fail") } - tree = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0) + require.NoError(err) targetVersion, _ := tree.LoadVersionForOverwriting(int64(maxLength * 2)) require.Equal(targetVersion, int64(maxLength), "targetVersion shouldn't larger than the actual tree latest version") - tree = NewMutableTree(mdb, 0) - _, err := tree.LoadVersionForOverwriting(int64(maxLength / 2)) + tree, err = NewMutableTree(mdb, 0) + require.NoError(err) + _, err = tree.LoadVersionForOverwriting(int64(maxLength / 2)) require.NoError(err, "LoadVersion should not fail") for version := 1; version <= maxLength/2; version++ { @@ -1260,7 +1303,8 @@ func TestLoadVersionForOverwriting(t *testing.T) { require.NoError(err, "SaveVersion should not fail, overwrite was allowed") // Reload tree at version 50, the latest tree version is 52 - tree = NewMutableTree(mdb, 0) + tree, err = NewMutableTree(mdb, 0) + require.NoError(err) _, err = tree.LoadVersion(int64(maxLength / 2)) require.NoError(err, "LoadVersion should not fail") @@ -1296,7 +1340,8 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { defer d.Close() defer os.RemoveAll("./bench.db") - tree := NewMutableTree(d, 0) + tree, err := NewMutableTree(d, 0) + require.NoError(b, err) for v := 1; v < numVersions; v++ { for i := 0; i < numKeysPerVersion; i++ { tree.Set([]byte(cmn.RandStr(16)), cmn.RandBytes(32)) @@ -1307,7 +1352,8 @@ func BenchmarkTreeLoadAndDelete(b *testing.B) { b.Run("LoadAndDelete", func(b *testing.B) { for n := 0; n < b.N; n++ { b.StopTimer() - tree = NewMutableTree(d, 0) + tree, err = NewMutableTree(d, 0) + require.NoError(b, err) runtime.GC() b.StartTimer() From 533652ce30f66effb29cce856639db5421095699 Mon Sep 17 00:00:00 2001 From: Timothy Chen Date: Fri, 20 Dec 2019 10:37:32 -0800 Subject: [PATCH 52/57] Fix build --- benchmarks/bench_test.go | 4 +++- cmd/iaviewer/main.go | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 6f7561524..42b9e9b6a 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -3,6 +3,7 @@ package benchmarks import ( "fmt" + "github.com/stretchr/testify/require" "math/rand" "os" "runtime" @@ -22,7 +23,8 @@ func randBytes(length int) []byte { } func prepareTree(b *testing.B, snapdb db.DB, memdb db.DB, keepEvery int64, keepRecent int64, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) { - t := iavl.NewMutableTreeWithOpts(snapdb, memdb, size, iavl.PruningOptions(keepEvery, keepRecent)) + t, err := iavl.NewMutableTreeWithOpts(snapdb, memdb, size, iavl.PruningOptions(keepEvery, keepRecent)) + require.NoError(b, err) keys := make([][]byte, size) for i := 0; i < size; i++ { diff --git a/cmd/iaviewer/main.go b/cmd/iaviewer/main.go index dd8d1a26b..230afb50f 100644 --- a/cmd/iaviewer/main.go +++ b/cmd/iaviewer/main.go @@ -99,7 +99,10 @@ func ReadTree(dir string, version int) (*iavl.MutableTree, error) { if err != nil { return nil, err } - tree := iavl.NewMutableTree(db, DefaultCacheSize) + tree, err := iavl.NewMutableTree(db, DefaultCacheSize) + if err != nil { + return nil, err + } ver, err := tree.LoadVersion(int64(version)) fmt.Printf("Got version: %d\n", ver) return tree, err From c5298552fe830054b4092626cfb90502606297de Mon Sep 17 00:00:00 2001 From: Timothy Chen Date: Fri, 20 Dec 2019 10:49:25 -0800 Subject: [PATCH 53/57] Fix linter --- mutable_tree.go | 18 +++++++++--------- pruning_test.go | 17 ++++++++--------- testutils_test.go | 2 +- tree_fuzz_test.go | 2 +- tree_test.go | 2 +- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index 5b80aef23..7c6f2a491 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -32,15 +32,15 @@ func NewMutableTree(db dbm.DB, cacheSize int) (*MutableTree, error) { func validateOptions(opts *Options) error { switch { - case opts == nil: - return nil - case opts.KeepEvery < 0: - return errors.New("keep every cannot be negative") - case opts.KeepRecent < 0: - return errors.New("keep recent cannot be negative") - case opts.KeepRecent == 0 && opts.KeepEvery > 1: - // We cannot snapshot more than every one version when we don't keep any versions in memory. - return errors.New("keep recent cannot be zero when keep every is set larger than one") + case opts == nil: + return nil + case opts.KeepEvery < 0: + return errors.New("keep every cannot be negative") + case opts.KeepRecent < 0: + return errors.New("keep recent cannot be negative") + case opts.KeepRecent == 0 && opts.KeepEvery > 1: + // We cannot snapshot more than every one version when we don't keep any versions in memory. + return errors.New("keep recent cannot be zero when keep every is set larger than one") } return nil diff --git a/pruning_test.go b/pruning_test.go index affe10762..941bbc431 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -41,7 +41,7 @@ func TestSave(t *testing.T) { binary.BigEndian.PutUint64(val, uint64(rand.Int63())) mt.Set(key, val) } - _, _, err := mt.SaveVersion() + _, _, err = mt.SaveVersion() require.Nil(t, err, "SaveVersion failed") } @@ -54,9 +54,9 @@ func TestSave(t *testing.T) { "Version: %d should not exist. KeepEvery: %d, KeepRecent: %d", v, PruningOptions(keepEvery, keepRecent)) // check that root exists in nodeDB - lv, err := mt.LazyLoadVersion(ver) + lv, lverr := mt.LazyLoadVersion(ver) require.Equal(t, ver, lv, "Version returned by LazyLoadVersion is wrong") - require.Nil(t, err, "Version should exist in nodeDB") + require.Nil(t, lverr, "Version should exist in nodeDB") } // check all expected versions are available. @@ -205,7 +205,7 @@ func TestRemoveKeys(t *testing.T) { for i := 0; i < 10; i++ { mt.Set([]byte(fmt.Sprintf("v%d:%d", v, i)), []byte(fmt.Sprintf("Val:v:%d:%d", v, i))) } - _, _, err := mt.SaveVersion() + _, _, err = mt.SaveVersion() require.NoError(t, err) } @@ -217,7 +217,7 @@ func TestRemoveKeys(t *testing.T) { _, removed := mt.Remove([]byte(key)) require.True(t, removed, "Key %s could not be removed", key) } - _, _, err := mt.SaveVersion() + _, _, err = mt.SaveVersion() require.NoError(t, err) } @@ -372,7 +372,7 @@ func TestSanity3(t *testing.T) { val = []byte(fmt.Sprintf("Val:v%d:i%d", i, j)) } mt.Set(key, val) - _, _, err := mt.SaveVersion() + _, _, err = mt.SaveVersion() if int64(i+1)%keepEvery == 0 { numSnapNodes += mt.nodeSize() } @@ -400,7 +400,7 @@ func TestNoSnapshots(t *testing.T) { db, mdb, close := getTestDBs() defer close() - keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB + keepRecent := rand.Int63n(8) + 2 //keep at least 2 versions in memDB mt, err := NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(0, keepRecent)) // test no snapshots require.NoError(t, err) @@ -485,7 +485,6 @@ func TestValidationOptions(t *testing.T) { require.Error(t, err) _, err = NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(-1, 0)) require.Error(t, err) - _, err = NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(0, -1)) + _, err = NewMutableTreeWithOpts(db, mdb, 5, PruningOptions(-1, -1)) require.Error(t, err) } - diff --git a/testutils_test.go b/testutils_test.go index 167443ab6..5fd284753 100644 --- a/testutils_test.go +++ b/testutils_test.go @@ -4,12 +4,12 @@ package iavl import ( "bytes" "fmt" - "github.com/stretchr/testify/require" "runtime" "testing" mrand "math/rand" + "github.com/stretchr/testify/require" amino "github.com/tendermint/go-amino" cmn "github.com/tendermint/iavl/common" db "github.com/tendermint/tm-db" diff --git a/tree_fuzz_test.go b/tree_fuzz_test.go index db58bf456..9ad471323 100644 --- a/tree_fuzz_test.go +++ b/tree_fuzz_test.go @@ -3,10 +3,10 @@ package iavl import ( "fmt" - "github.com/stretchr/testify/require" "math/rand" "testing" + "github.com/stretchr/testify/require" cmn "github.com/tendermint/iavl/common" ) diff --git a/tree_test.go b/tree_test.go index 364cd871b..1ac54b2bd 100644 --- a/tree_test.go +++ b/tree_test.go @@ -1270,7 +1270,7 @@ func TestLoadVersionForOverwriting(t *testing.T) { countStr := strconv.Itoa(count) // Set one kv pair and save version tree.Set([]byte("key"+countStr), []byte("value"+countStr)) - _, _, err := tree.SaveVersion() + _, _, err = tree.SaveVersion() require.NoError(err, "SaveVersion should not fail") } From bbb6aa3ba098e362b09c3e10f450f75513d08e9b Mon Sep 17 00:00:00 2001 From: Marko Baricevic Date: Wed, 8 Jan 2020 10:50:43 +0100 Subject: [PATCH 54/57] move pruning docs --- PRUNING.md => docs/tree/PRUNING.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename PRUNING.md => docs/tree/PRUNING.md (100%) diff --git a/PRUNING.md b/docs/tree/PRUNING.md similarity index 100% rename from PRUNING.md rename to docs/tree/PRUNING.md From 72d9cdd26ef6947b50fd17f9c7b4f95db553ba53 Mon Sep 17 00:00:00 2001 From: Timothy Chen Date: Fri, 10 Jan 2020 14:38:34 -0800 Subject: [PATCH 55/57] Update pruning docs and tests --- benchmarks/prune_test.go | 35 +++++++++++++++++------------------ docs/tree/PRUNING.md | 5 ++++- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index f2b1dcf62..288c0d906 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -32,10 +32,10 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 snapDB := db.NewDB("test", "goleveldb", dirName) defer snapDB.Close() - // var mem runtime.MemStats - // runtime.ReadMemStats(&mem) - // memSize := mem.Alloc - // maxVersion := 0 + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + memSize := mem.Alloc + //maxVersion := 0 var keys [][]byte for i := 0; i < 100; i++ { keys = append(keys, randBytes(keyLen)) @@ -56,20 +56,19 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 if err != nil { b.Errorf("Can't save version %d: %v", i, err) } - // // Pause timer to garbage-collect and remeasure memory usage - // b.StopTimer() - // runtime.GC() - // runtime.ReadMemStats(&mem) - // // update memSize if it has increased after saveVersion - // if memSize < mem.Alloc { - // memSize = mem.Alloc - // maxVersion = i - // } - // b.StartTimer() + + // Pause timer to garbage-collect and remeasure memory usage b.StopTimer() runtime.GC() + runtime.ReadMemStats(&mem) + // update memSize if it has increased after saveVersion + if memSize < mem.Alloc { + memSize = mem.Alloc + maxVersion = i + } b.StartTimer() } + //fmt.Printf("Maxmimum Memory usage was %0.2f MB at height %d\n", float64(memSize)/1000000, maxVersion) b.StopTimer() } @@ -77,16 +76,16 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 func BenchmarkPruningStrategies(b *testing.B) { ps := []pruningstrat{ {1, 0}, // default pruning strategy - //{1, 1}, + {1, 1}, {0, 1}, // keep single recent version {100, 1}, {100, 5}, // simple pruning {5, 1}, {5, 2}, {10, 2}, - // {1000, 10}, // average pruning - // {1000, 1}, // extreme pruning - // {10000, 100}, // SDK pruning + {1000, 10}, // average pruning + {1000, 1}, // extreme pruning + {10000, 100}, // SDK pruning } for _, ps := range ps { ps := ps diff --git a/docs/tree/PRUNING.md b/docs/tree/PRUNING.md index 346a18980..5fc620262 100644 --- a/docs/tree/PRUNING.md +++ b/docs/tree/PRUNING.md @@ -4,7 +4,6 @@ Setting Pruning fields in the IAVL tree can optimize performance by only writing We can set custom pruning fields in IAVL using: `NewMutableTreeWithOpts` - ## Current design ### NodeDB @@ -22,6 +21,10 @@ keepRecent int64 // Saves recent versions in memory If version is not going to be persisted to disk, the version is simply saved in `recentDB` (typically a `memDB`) If version is persisted to disk, the version is written to `recentDB` **and** `snapshotDB` (typically `levelDB`) +For example, setting keepEvery to 1 and keepRecent to 0 (which is the default setting) will persist every version to snapshot and skip storing +anything in memDB. Setting keepEvery to 10000 and keepRecent to 100 (default Cosmos SDK pruning strategy) is snapshotting every 10000th version and will keep the last 100 versions +in memDB. + #### Orphans: Save orphan to `memDB` under `o|toVersion|fromVersion`. From a382f71ca1cd54248d2c96809359ad581cfc550e Mon Sep 17 00:00:00 2001 From: Timothy Chen Date: Tue, 14 Jan 2020 11:30:34 -0800 Subject: [PATCH 56/57] Fix build --- benchmarks/prune_test.go | 2 +- go.mod | 2 +- go.sum | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/benchmarks/prune_test.go b/benchmarks/prune_test.go index 288c0d906..dfc295ba3 100644 --- a/benchmarks/prune_test.go +++ b/benchmarks/prune_test.go @@ -64,7 +64,7 @@ func runBlockChain(b *testing.B, prefix string, keepEvery int64, keepRecent int6 // update memSize if it has increased after saveVersion if memSize < mem.Alloc { memSize = mem.Alloc - maxVersion = i + //maxVersion = i } b.StartTimer() } diff --git a/go.mod b/go.mod index 224d7eb9d..2fd10dc5a 100644 --- a/go.mod +++ b/go.mod @@ -8,5 +8,5 @@ require ( github.com/tendermint/go-amino v0.14.1 github.com/tendermint/tendermint v0.32.9 github.com/tendermint/tm-db v0.4.0 - golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a + golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 ) diff --git a/go.sum b/go.sum index 8c0d535d6..ee8bf85c3 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a h1:YX8ljsm6wXlHZO+aRz9Exqr0evNhKRNe5K/gi+zKh4U= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0= +golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -203,6 +205,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -219,6 +222,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= From 0b86aa06a74a5b0f19848c028c1694eea89c45a4 Mon Sep 17 00:00:00 2001 From: Marko Date: Thu, 16 Jan 2020 12:10:39 +0100 Subject: [PATCH 57/57] errors: add some error handling (#199) * errors: add some error handling tm-db 0.4.0 introduced more returns with errors, bubbling up more errors because of this. Signed-off-by: Marko Baricevic * wrap errors --- mutable_tree.go | 37 ++++++++++++++++++----- mutable_tree_test.go | 3 +- nodedb.go | 71 +++++++++++++++++++++++++++++++------------- pruning_test.go | 6 ++-- 4 files changed, 86 insertions(+), 31 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index d06573678..82071c8ea 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -464,15 +464,25 @@ func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { panic(err) } } - tree.ndb.Commit() + err := tree.ndb.Commit() + if err != nil { + return nil, version, err + } // Prune nodeDB and delete any pruned versions from tree.versions - prunedVersions := tree.ndb.PruneRecentVersions() + prunedVersions, err := tree.ndb.PruneRecentVersions() + if err != nil { + return nil, version, err + } for _, pVer := range prunedVersions { delete(tree.versions, pVer) } - tree.ndb.Commit() + err = tree.ndb.Commit() + if err != nil { + return nil, version, err + } + tree.version = version tree.versions[version] = true @@ -498,8 +508,15 @@ func (tree *MutableTree) DeleteVersion(version int64) error { return errors.Wrap(ErrVersionDoesNotExist, "") } - tree.ndb.DeleteVersion(version, true) - tree.ndb.Commit() + err := tree.ndb.DeleteVersion(version, true) + if err != nil { + return err + } + + err = tree.ndb.Commit() + if err != nil { + return err + } delete(tree.versions, version) @@ -518,10 +535,16 @@ func (tree *MutableTree) deleteVersionsFrom(version int64) error { if version == tree.version { return errors.Errorf("cannot delete latest saved version (%d)", version) } - tree.ndb.DeleteVersion(version, false) + err := tree.ndb.DeleteVersion(version, false) + if err != nil { + return err + } delete(tree.versions, version) } - tree.ndb.Commit() + err := tree.ndb.Commit() + if err != nil { + return err + } tree.ndb.resetLatestVersion(newLatestVersion) return nil } diff --git a/mutable_tree_test.go b/mutable_tree_test.go index 1cad402ad..c2b29d965 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -27,7 +27,8 @@ func TestDelete(t *testing.T) { require.Nil(t, k1Value) key := tree.ndb.rootKey(version) - memDb.Set(key, hash) + err = memDb.Set(key, hash) + require.NoError(t, err) tree.versions[version] = true k1Value, _, err = tree.GetVersionedWithProof([]byte("k1"), version) diff --git a/nodedb.go b/nodedb.go index c4a5610d5..0dbeb9d3b 100644 --- a/nodedb.go +++ b/nodedb.go @@ -7,6 +7,8 @@ import ( "sort" "sync" + "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto/tmhash" dbm "github.com/tendermint/tm-db" ) @@ -159,7 +161,7 @@ func (ndb *nodeDB) Has(hash []byte) (bool, error) { val, err := ndb.recentDB.Get(key) if err != nil { - return false, err + return false, errors.Wrap(err, "recentDB") } if val != nil { return true, nil @@ -168,13 +170,13 @@ func (ndb *nodeDB) Has(hash []byte) (bool, error) { if ldb, ok := ndb.snapshotDB.(*dbm.GoLevelDB); ok { exists, err := ldb.DB().Has(key, nil) if err != nil { - return false, err + return false, errors.Wrap(err, "snapshotDB") } return exists, nil } value, err := ndb.snapshotDB.Get(key) if err != nil { - return false, err + return false, errors.Wrap(err, "snapshotDB") } return value != nil, nil } @@ -218,20 +220,24 @@ func (ndb *nodeDB) SaveBranch(node *Node, flushToDisk bool) []byte { } // DeleteVersion deletes a tree version from disk. -func (ndb *nodeDB) DeleteVersion(version int64, checkLatestVersion bool) { - ndb.deleteVersion(version, checkLatestVersion, false) +func (ndb *nodeDB) DeleteVersion(version int64, checkLatestVersion bool) error { + return ndb.deleteVersion(version, checkLatestVersion, false) } -func (ndb *nodeDB) DeleteVersionFromRecent(version int64, checkLatestVersion bool) { - ndb.deleteVersion(version, checkLatestVersion, true) +func (ndb *nodeDB) DeleteVersionFromRecent(version int64, checkLatestVersion bool) error { + return ndb.deleteVersion(version, checkLatestVersion, true) } -func (ndb *nodeDB) deleteVersion(version int64, checkLatestVersion, memOnly bool) { +func (ndb *nodeDB) deleteVersion(version int64, checkLatestVersion, memOnly bool) error { ndb.mtx.Lock() defer ndb.mtx.Unlock() - ndb.deleteOrphans(version, memOnly) + err := ndb.deleteOrphans(version, memOnly) + if err != nil { + return err + } ndb.deleteRoot(version, checkLatestVersion, memOnly) + return nil } // Saves orphaned nodes to disk under a special prefix. @@ -273,17 +279,22 @@ func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64, flushTo // deleteOrphans deletes orphaned nodes from disk, and the associated orphan // entries. -func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) { +func (ndb *nodeDB) deleteOrphans(version int64, memOnly bool) error { if ndb.opts.KeepRecent != 0 { ndb.deleteOrphansMem(version) } + var err error if ndb.isSnapshotVersion(version) && !memOnly { predecessor := getPreviousVersionFromDB(version, ndb.snapshotDB) traverseOrphansVersionFromDB(ndb.snapshotDB, version, func(key, hash []byte) { - ndb.snapshotDB.Delete(key) + err = ndb.snapshotDB.Delete(key) ndb.deleteOrphansHelper(ndb.snapshotDB, ndb.snapshotBatch, true, predecessor, key, hash) }) } + if err != nil { + return errors.Wrap(err, "snapshotDB err in delete") + } + return nil } func (ndb *nodeDB) deleteOrphansMem(version int64) { @@ -331,16 +342,20 @@ func (ndb *nodeDB) deleteOrphansHelper(db dbm.DB, batch dbm.Batch, flushToDisk b } } -func (ndb *nodeDB) PruneRecentVersions() (prunedVersions []int64) { +func (ndb *nodeDB) PruneRecentVersions() (prunedVersions []int64, err error) { if ndb.opts.KeepRecent == 0 || ndb.latestVersion-ndb.opts.KeepRecent <= 0 { - return nil + return nil, nil } pruneVer := ndb.latestVersion - ndb.opts.KeepRecent - ndb.DeleteVersionFromRecent(pruneVer, true) + err = ndb.DeleteVersionFromRecent(pruneVer, true) + if err != nil { + return nil, err + } + if ndb.isSnapshotVersion(pruneVer) { - return nil + return nil, nil } - return append(prunedVersions, pruneVer) + return append(prunedVersions, pruneVer), nil } func (ndb *nodeDB) nodeKey(hash []byte) []byte { @@ -535,28 +550,42 @@ func (ndb *nodeDB) cacheNode(node *Node) { } // Write to disk and memDB -func (ndb *nodeDB) Commit() { +func (ndb *nodeDB) Commit() error { ndb.mtx.Lock() defer ndb.mtx.Unlock() + var err error if ndb.opts.KeepEvery != 0 { if ndb.opts.Sync { - ndb.snapshotBatch.WriteSync() + err = ndb.snapshotBatch.WriteSync() + if err != nil { + return errors.Wrap(err, "error in snapShotBatch writesync") + } } else { - ndb.snapshotBatch.Write() + err = ndb.snapshotBatch.Write() + if err != nil { + return errors.Wrap(err, "error in snapShotBatch write") + } } ndb.snapshotBatch.Close() } if ndb.opts.KeepRecent != 0 { if ndb.opts.Sync { - ndb.recentBatch.WriteSync() + err = ndb.recentBatch.WriteSync() + if err != nil { + return errors.Wrap(err, "error in recentBatch writesync") + } } else { - ndb.recentBatch.Write() + err = ndb.recentBatch.Write() + if err != nil { + return errors.Wrap(err, "error in recentBatch write") + } } ndb.recentBatch.Close() } ndb.snapshotBatch = ndb.snapshotDB.NewBatch() ndb.recentBatch = ndb.recentDB.NewBatch() + return nil } func (ndb *nodeDB) getRoot(version int64) ([]byte, error) { diff --git a/pruning_test.go b/pruning_test.go index 941bbc431..dacbebef7 100644 --- a/pruning_test.go +++ b/pruning_test.go @@ -289,8 +289,10 @@ func TestSanity1(t *testing.T) { require.Equal(t, mt.nodeSize(), len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), "SnapshotDB did not save correctly") for i := 1; i < 5; i++ { - mt.ndb.DeleteVersionFromRecent(int64(i), true) - mt.ndb.Commit() + err := mt.ndb.DeleteVersionFromRecent(int64(i), true) + require.NoError(t, err) + err = mt.ndb.Commit() + require.NoError(t, err) } require.Equal(t, len(mt.ndb.nodesFromDB(mt.ndb.snapshotDB)), len(mt.ndb.nodesFromDB(mt.ndb.recentDB)), "DB sizes should be the same")