diff --git a/README.md b/README.md index 835d0cf0f..cc5a0e5cc 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ ## IAVL+ Tree -A snapshottable (immutable) AVL+ tree for persistent data +**Note: Requires Go 1.8+** -**Note** Please make sure you read the [caveat](https://github.com/tendermint/merkleeyes/blob/develop/iavl/iavl_tree.go#L34-L40) on `Copy`. If you have a backing DB and call `Save` to persist the state, all existing copies become potentially invalid and may panic if used. For safe coding, you must throw away all references upon save, and `Copy` again from the new, committed state. +A versioned, snapshottable (immutable) AVL+ tree for persistent data. The purpose of this data structure is to provide persistent storage for key-value pairs (say to store account balances) such that a deterministic merkle root hash can be computed. The tree is balanced using a variant of the [AVL algortihm](http://en.wikipedia.org/wiki/AVL_tree) so all operations are O(log(n)). -Nodes of this tree are immutable and indexed by its hash. Thus any node serves as an immutable snapshot which lets us stage uncommitted transactions from the mempool cheaply, and we can instantly roll back to the last committed state to process transactions of a newly committed block (which may not be the same set of transactions as those from the mempool). +Nodes of this tree are immutable and indexed by their hash. Thus any node serves as an immutable snapshot which lets us stage uncommitted transactions from the mempool cheaply, and we can instantly roll back to the last committed state to process transactions of a newly committed block (which may not be the same set of transactions as those from the mempool). In an AVL tree, the heights of the two child subtrees of any node differ by at most one. Whenever this condition is violated upon an update, the tree is rebalanced by creating O(log(n)) new nodes that point to unmodified nodes of the old tree. In the original AVL algorithm, inner nodes can also hold key-value pairs. The AVL+ algorithm (note the plus) modifies the AVL algorithm to keep all values on leaf nodes, while only using branch-nodes to store keys. This simplifies the algorithm while keeping the merkle hash trail short. diff --git a/iavl_test.go b/basic_test.go similarity index 64% rename from iavl_test.go rename to basic_test.go index ee178c0e1..49aa196fb 100644 --- a/iavl_test.go +++ b/basic_test.go @@ -2,85 +2,18 @@ package iavl import ( "bytes" - "fmt" mrand "math/rand" "sort" + "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/go-wire" - . "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/db" - . "github.com/tendermint/tmlibs/test" - - "runtime" - "testing" ) -const testReadLimit = 1 << 20 // Some reasonable limit for wire.Read*() lmt - -func randstr(length int) string { - return RandStr(length) -} - -func i2b(i int) []byte { - bz := make([]byte, 4) - wire.PutInt32(bz, int32(i)) - return bz -} - -func b2i(bz []byte) int { - i := wire.GetInt32(bz) - return int(i) -} - -// Convenience for a new node -func N(l, r interface{}) *IAVLNode { - var left, right *IAVLNode - if _, ok := l.(*IAVLNode); ok { - left = l.(*IAVLNode) - } else { - left = NewIAVLNode(i2b(l.(int)), nil) - } - if _, ok := r.(*IAVLNode); ok { - right = r.(*IAVLNode) - } else { - right = NewIAVLNode(i2b(r.(int)), nil) - } - - n := &IAVLNode{ - key: right.lmd(nil).key, - value: nil, - leftNode: left, - rightNode: right, - } - n.calcHeightAndSize(nil) - return n -} - -// Setup a deep node -func T(n *IAVLNode) *IAVLTree { - d := db.NewDB("test", db.MemDBBackendStr, "") - t := NewIAVLTree(0, d) - - n.hashWithCount() - t.root = n - return t -} - -// Convenience for simple printing of keys & tree structure -func P(n *IAVLNode) string { - if n.height == 0 { - return fmt.Sprintf("%v", b2i(n.key)) - } else { - return fmt.Sprintf("(%v %v)", P(n.leftNode), P(n.rightNode)) - } -} - func TestBasic(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) - var up bool - up = tree.Set([]byte("1"), []byte("one")) + var tree *Tree = NewTree(0, nil) + up := tree.Set([]byte("1"), []byte("one")) if up { t.Error("Did not expect an update (should have been create)") } @@ -99,8 +32,8 @@ func TestBasic(t *testing.T) { // Test 0x00 { - idx, val, exists := tree.Get([]byte{0x00}) - if exists { + idx, val := tree.Get([]byte{0x00}) + if val != nil { t.Errorf("Expected no value to exist") } if idx != 0 { @@ -113,8 +46,8 @@ func TestBasic(t *testing.T) { // Test "1" { - idx, val, exists := tree.Get([]byte("1")) - if !exists { + idx, val := tree.Get([]byte("1")) + if val == nil { t.Errorf("Expected value to exist") } if idx != 0 { @@ -127,8 +60,8 @@ func TestBasic(t *testing.T) { // Test "2" { - idx, val, exists := tree.Get([]byte("2")) - if !exists { + idx, val := tree.Get([]byte("2")) + if val == nil { t.Errorf("Expected value to exist") } if idx != 1 { @@ -141,8 +74,8 @@ func TestBasic(t *testing.T) { // Test "4" { - idx, val, exists := tree.Get([]byte("4")) - if exists { + idx, val := tree.Get([]byte("4")) + if val != nil { t.Errorf("Expected no value to exist") } if idx != 2 { @@ -155,8 +88,8 @@ func TestBasic(t *testing.T) { // Test "6" { - idx, val, exists := tree.Get([]byte("6")) - if exists { + idx, val := tree.Get([]byte("6")) + if val != nil { t.Errorf("Expected no value to exist") } if idx != 3 { @@ -170,29 +103,29 @@ func TestBasic(t *testing.T) { func TestUnit(t *testing.T) { - expectHash := func(tree *IAVLTree, hashCount int) { + expectHash := func(tree *Tree, hashCount int) { // ensure number of new hash calculations is as expected. - hash, count := tree.HashWithCount() + hash, count := tree.hashWithCount() if count != hashCount { t.Fatalf("Expected %v new hashes, got %v", hashCount, count) } // nuke hashes and reconstruct hash, ensure it's the same. - tree.root.traverse(tree, true, func(node *IAVLNode) bool { + tree.root.traverse(tree, true, func(node *Node) bool { node.hash = nil return false }) // ensure that the new hash after nuking is the same as the old. - newHash, _ := tree.HashWithCount() - if bytes.Compare(hash, newHash) != 0 { + newHash, _ := tree.hashWithCount() + if !bytes.Equal(hash, newHash) { t.Fatalf("Expected hash %v but got %v after nuking", hash, newHash) } } - expectSet := func(tree *IAVLTree, i int, repr string, hashCount int) { + expectSet := func(tree *Tree, i int, repr string, hashCount int) { origNode := tree.root - updated := tree.Set(i2b(i), nil) + updated := tree.Set(i2b(i), []byte{}) // ensure node was added & structure is as expected. - if updated == true || P(tree.root) != repr { + if updated || P(tree.root) != repr { t.Fatalf("Adding %v to %v:\nExpected %v\nUnexpectedly got %v updated:%v", i, P(origNode), repr, P(tree.root), updated) } @@ -201,7 +134,7 @@ func TestUnit(t *testing.T) { tree.root = origNode } - expectRemove := func(tree *IAVLTree, i int, repr string, hashCount int) { + expectRemove := func(tree *Tree, i int, repr string, hashCount int) { origNode := tree.root value, removed := tree.Remove(i2b(i)) // ensure node was added & structure is as expected. @@ -251,20 +184,13 @@ func TestUnit(t *testing.T) { } -func randBytes(length int) []byte { - key := make([]byte, length) - // math.rand.Read always returns err=nil - mrand.Read(key) - return key -} - func TestRemove(t *testing.T) { size := 10000 keyLen, dataLen := 16, 40 d := db.NewDB("test", "memdb", "") defer d.Close() - t1 := NewIAVLTree(size, d) + t1 := NewVersionedTree(size, d) // insert a bunch of random nodes keys := make([][]byte, size) @@ -282,10 +208,8 @@ func TestRemove(t *testing.T) { key := keys[mrand.Int31n(l)] t1.Remove(key) } - // FIXME: this Save() causes a panic! - t1.Save() + t1.SaveVersion(uint64(i)) } - } func TestIntegration(t *testing.T) { @@ -296,7 +220,7 @@ func TestIntegration(t *testing.T) { } records := make([]*record, 400) - var tree *IAVLTree = NewIAVLTree(0, nil) + var tree *Tree = NewTree(0, nil) randomRecord := func() *record { return &record{randstr(20), randstr(20)} @@ -305,9 +229,7 @@ func TestIntegration(t *testing.T) { for i := range records { r := randomRecord() records[i] = r - //t.Log("New record", r) - //PrintIAVLNode(tree.root) - updated := tree.Set([]byte(r.key), nil) + updated := tree.Set([]byte(r.key), []byte{}) if updated { t.Error("should have not been updated") } @@ -327,7 +249,7 @@ func TestIntegration(t *testing.T) { if has := tree.Has([]byte(randstr(12))); has { t.Error("Table has extra key") } - if _, val, _ := tree.Get([]byte(r.key)); string(val) != string(r.value) { + if _, val := tree.Get([]byte(r.key)); string(val) != string(r.value) { t.Error("wrong value") } } @@ -345,7 +267,7 @@ func TestIntegration(t *testing.T) { if has := tree.Has([]byte(randstr(12))); has { t.Error("Table has extra key") } - _, val, _ := tree.Get([]byte(r.key)) + _, val := tree.Get([]byte(r.key)) if string(val) != string(r.value) { t.Error("wrong value") } @@ -380,7 +302,7 @@ func TestIterateRange(t *testing.T) { } sort.Strings(keys) - var tree *IAVLTree = NewIAVLTree(0, nil) + var tree *Tree = NewTree(0, nil) // insert all the data for _, r := range records { @@ -435,34 +357,6 @@ func TestIterateRange(t *testing.T) { trav = traverser{} tree.IterateRange([]byte("g"), nil, false, trav.view) expectTraverse(t, trav, "low", "good", 2) - -} - -type traverser struct { - first string - last string - count int -} - -func (t *traverser) view(key, value []byte) bool { - if t.first == "" { - t.first = string(key) - } - t.last = string(key) - t.count += 1 - return false -} - -func expectTraverse(t *testing.T, trav traverser, start, end string, count int) { - if trav.first != start { - t.Error("Bad start", start, trav.first) - } - if trav.last != end { - t.Error("Bad end", end, trav.last) - } - if trav.count != count { - t.Error("Bad count", count, trav.count) - } } func TestPersistence(t *testing.T) { @@ -475,63 +369,36 @@ func TestPersistence(t *testing.T) { } // Construct some tree and save it - t1 := NewIAVLTree(0, db) + t1 := NewVersionedTree(0, db) for key, value := range records { t1.Set([]byte(key), []byte(value)) } - t1.Save() - - hash, _ := t1.HashWithCount() + t1.SaveVersion(1) // Load a tree - t2 := NewIAVLTree(0, db) - t2.Load(hash) + t2 := NewVersionedTree(0, db) + t2.Load() for key, value := range records { - _, t2value, _ := t2.Get([]byte(key)) + _, t2value := t2.Get([]byte(key)) if string(t2value) != value { t.Fatalf("Invalid value. Expected %v, got %v", value, t2value) } } } -func testProof(t *testing.T, proof *KeyExistsProof, keyBytes, valueBytes, rootHashBytes []byte) { - // Proof must verify. - require.NoError(t, proof.Verify(keyBytes, valueBytes, rootHashBytes)) - - // Write/Read then verify. - proofBytes := wire.BinaryBytes(proof) - proof2, err := ReadKeyExistsProof(proofBytes) - require.Nil(t, err, "Failed to read KeyExistsProof from bytes: %v", err) - require.NoError(t, proof2.Verify(keyBytes, valueBytes, proof.RootHash)) - - // Random mutations must not verify - for i := 0; i < 10; i++ { - badProofBytes := MutateByteSlice(proofBytes) - badProof, err := ReadKeyExistsProof(badProofBytes) - // may be invalid... errors are okay - if err == nil { - assert.Error(t, badProof.Verify(keyBytes, valueBytes, rootHashBytes), - "Proof was still valid after a random mutation:\n%X\n%X", - proofBytes, badProofBytes) - } - } - - // targetted changes fails... - proof.RootHash = MutateByteSlice(proof.RootHash) - assert.Error(t, proof.Verify(keyBytes, valueBytes, rootHashBytes)) -} +func TestProof(t *testing.T) { + t.Skipf("This test has a race condition causing it to occasionally panic.") -func TestIAVLProof(t *testing.T) { // Construct some random tree db := db.NewMemDB() - var tree *IAVLTree = NewIAVLTree(100, db) + var tree *VersionedTree = NewVersionedTree(100, db) for i := 0; i < 1000; i++ { key, value := randstr(20), randstr(20) tree.Set([]byte(key), []byte(value)) } // Persist the items so far - tree.Save() + tree.SaveVersion(1) // Add more items so it's not all persisted for i := 0; i < 100; i++ { @@ -551,9 +418,9 @@ func TestIAVLProof(t *testing.T) { }) } -func TestIAVLTreeProof(t *testing.T) { +func TestTreeProof(t *testing.T) { db := db.NewMemDB() - var tree *IAVLTree = NewIAVLTree(100, db) + var tree *Tree = NewTree(100, db) // should get false for proof with nil root _, _, err := tree.GetWithProof([]byte("foo")) @@ -576,43 +443,8 @@ func TestIAVLTreeProof(t *testing.T) { for _, key := range keys { value, proof, err := tree.GetWithProof(key) if assert.NoError(t, err) { - require.Nil(t, err, "Failed to read IAVLProof from bytes: %v", err) + require.Nil(t, err, "Failed to read proof from bytes: %v", err) assert.NoError(t, proof.Verify(key, value, root)) } } } - -func BenchmarkImmutableAvlTreeMemDB(b *testing.B) { - db := db.NewDB("test", db.MemDBBackendStr, "") - benchmarkImmutableAvlTreeWithDB(b, db) -} - -func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { - defer db.Close() - - b.StopTimer() - - t := NewIAVLTree(100000, db) - for i := 0; i < 1000000; i++ { - t.Set(i2b(int(RandInt32())), nil) - if i > 990000 && i%1000 == 999 { - t.Save() - } - } - b.ReportAllocs() - t.Save() - - fmt.Println("ok, starting") - - runtime.GC() - - b.StartTimer() - for i := 0; i < b.N; i++ { - ri := i2b(int(RandInt32())) - t.Set(ri, nil) - t.Remove(ri) - if i%100 == 99 { - t.Save() - } - } -} diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index 45a44e003..e4e9f8040 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -18,8 +18,8 @@ func randBytes(length int) []byte { return key } -func prepareTree(db db.DB, size, keyLen, dataLen int) (*iavl.IAVLTree, [][]byte) { - t := iavl.NewIAVLTree(size, db) +func prepareTree(db db.DB, size, keyLen, dataLen int) (*iavl.VersionedTree, [][]byte) { + t := iavl.NewVersionedTree(size, db) keys := make([][]byte, size) for i := 0; i < size; i++ { @@ -28,19 +28,19 @@ func prepareTree(db db.DB, size, keyLen, dataLen int) (*iavl.IAVLTree, [][]byte) keys[i] = key } t.Hash() - t.Save() + t.SaveVersion(t.LatestVersion() + 1) runtime.GC() return t, keys } -func runQueries(b *testing.B, t *iavl.IAVLTree, keyLen int) { +func runQueries(b *testing.B, t *iavl.VersionedTree, keyLen int) { for i := 0; i < b.N; i++ { q := randBytes(keyLen) t.Get(q) } } -func runKnownQueries(b *testing.B, t *iavl.IAVLTree, keys [][]byte) { +func runKnownQueries(b *testing.B, t *iavl.VersionedTree, keys [][]byte) { l := int32(len(keys)) for i := 0; i < b.N; i++ { q := keys[rand.Int31n(l)] @@ -48,31 +48,31 @@ func runKnownQueries(b *testing.B, t *iavl.IAVLTree, keys [][]byte) { } } -func runInsert(b *testing.B, t *iavl.IAVLTree, keyLen, dataLen, blockSize int) *iavl.IAVLTree { +func runInsert(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize int) *iavl.VersionedTree { for i := 1; i <= b.N; i++ { t.Set(randBytes(keyLen), randBytes(dataLen)) if i%blockSize == 0 { t.Hash() - t.Save() + t.SaveVersion(t.LatestVersion() + 1) } } return t } -func runUpdate(b *testing.B, t *iavl.IAVLTree, dataLen, blockSize int, keys [][]byte) *iavl.IAVLTree { +func runUpdate(b *testing.B, t *iavl.VersionedTree, dataLen, blockSize int, keys [][]byte) *iavl.VersionedTree { l := int32(len(keys)) for i := 1; i <= b.N; i++ { key := keys[rand.Int31n(l)] t.Set(key, randBytes(dataLen)) if i%blockSize == 0 { t.Hash() - t.Save() + t.SaveVersion(t.LatestVersion() + 1) } } return t } -func runDelete(b *testing.B, t *iavl.IAVLTree, blockSize int, keys [][]byte) *iavl.IAVLTree { +func runDelete(b *testing.B, t *iavl.VersionedTree, blockSize int, keys [][]byte) *iavl.VersionedTree { var key []byte l := int32(len(keys)) for i := 1; i <= b.N; i++ { @@ -82,19 +82,21 @@ func runDelete(b *testing.B, t *iavl.IAVLTree, blockSize int, keys [][]byte) *ia t.Remove(key) if i%blockSize == 0 { t.Hash() - t.Save() + t.SaveVersion(t.LatestVersion() + 1) } } return t } // runBlock measures time for an entire block, not just one tx -func runBlock(b *testing.B, t *iavl.IAVLTree, keyLen, dataLen, blockSize int, keys [][]byte) *iavl.IAVLTree { +func runBlock(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize int, keys [][]byte) *iavl.VersionedTree { l := int32(len(keys)) + // XXX: This was adapted to work with VersionedTree but needs to be re-thought. + lastCommit := t - real := t.Copy() - check := t.Copy() + real := t + check := t for i := 0; i < b.N; i++ { for j := 0; j < blockSize; j++ { @@ -116,10 +118,8 @@ func runBlock(b *testing.B, t *iavl.IAVLTree, keyLen, dataLen, blockSize int, ke // at the end of a block, move it all along.... real.Hash() - real.Save() + real.SaveVersion(real.LatestVersion() + 1) lastCommit = real - real = lastCommit.Copy() - check = lastCommit.Copy() } return lastCommit @@ -150,7 +150,6 @@ type benchmark struct { func BenchmarkMedium(b *testing.B) { benchmarks := []benchmark{ - {"nodb", 100000, 100, 16, 40}, {"memdb", 100000, 100, 16, 40}, {"goleveldb", 100000, 100, 16, 40}, // FIXME: this crashes on init! Either remove support, or make it work. @@ -162,7 +161,6 @@ func BenchmarkMedium(b *testing.B) { func BenchmarkSmall(b *testing.B) { benchmarks := []benchmark{ - {"nodb", 1000, 100, 4, 10}, {"memdb", 1000, 100, 4, 10}, {"goleveldb", 1000, 100, 4, 10}, // FIXME: this crashes on init! Either remove support, or make it work. @@ -174,7 +172,6 @@ func BenchmarkSmall(b *testing.B) { func BenchmarkLarge(b *testing.B) { benchmarks := []benchmark{ - {"nodb", 1000000, 100, 16, 40}, {"memdb", 1000000, 100, 16, 40}, {"goleveldb", 1000000, 100, 16, 40}, // FIXME: this crashes on init! Either remove support, or make it work. @@ -184,31 +181,6 @@ func BenchmarkLarge(b *testing.B) { runBenchmarks(b, benchmarks) } -func BenchmarkMemInitSizes(b *testing.B) { - benchmarks := []benchmark{ - {"nodb", 10000, 100, 16, 40}, - {"nodb", 70000, 100, 16, 40}, - {"nodb", 500000, 100, 16, 40}, - // This uses something like 1.5-2GB RAM - {"nodb", 3500000, 100, 16, 40}, - // This requires something like 5GB RAM and may crash on some AWS instances - {"nodb", 10000000, 100, 16, 40}, - } - runBenchmarks(b, benchmarks) -} - -func BenchmarkMemKeySizes(b *testing.B) { - benchmarks := []benchmark{ - {"nodb", 100000, 100, 4, 80}, - {"nodb", 100000, 100, 16, 80}, - {"nodb", 100000, 100, 32, 80}, - {"nodb", 100000, 100, 64, 80}, - {"nodb", 100000, 100, 128, 80}, - {"nodb", 100000, 100, 256, 80}, - } - runBenchmarks(b, benchmarks) -} - func BenchmarkLevelDBBatchSizes(b *testing.B) { benchmarks := []benchmark{ {"goleveldb", 100000, 5, 16, 40}, diff --git a/circle.yml b/circle.yml index c3f31c58e..2c4b02306 100644 --- a/circle.yml +++ b/circle.yml @@ -1,11 +1,21 @@ machine: environment: - GO15VENDOREXPERIMENT: 1 + GOPATH: /home/ubuntu/.go_workspace + REPO: $GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME + hosts: + circlehost: 127.0.0.1 + localhost: 127.0.0.1 + +checkout: + post: + - rm -rf $REPO + - mkdir -p $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME + - mv $HOME/$CIRCLE_PROJECT_REPONAME $REPO dependencies: - pre: - - make get_deps + override: + - "cd $REPO && make get_vendor_deps" test: override: - - make test + - "cd $REPO && make test" diff --git a/doc.go b/doc.go new file mode 100644 index 000000000..b347fc654 --- /dev/null +++ b/doc.go @@ -0,0 +1,45 @@ +// Basic usage of VersionedTree. +// +// import "github.com/tendermint/iavl" +// import "github.com/tendermint/tmlibs/db" +// ... +// +// tree := iavl.NewVersionedTree(128, db.NewMemDB()) +// +// tree.IsEmpty() // true +// +// tree.Set([]byte("alice"), []byte("abc")) +// tree.SaveVersion(1) +// +// tree.Set([]byte("alice"), []byte("xyz")) +// tree.Set([]byte("bob"), []byte("xyz")) +// tree.SaveVersion(2) +// +// tree.LatestVersion() // 2 +// +// tree.GetVersioned([]byte("alice"), 1) // "abc" +// tree.GetVersioned([]byte("alice"), 2) // "xyz" +// +// Proof of existence: +// +// root := tree.Hash() +// val, proof, err := tree.GetVersionedWithProof([]byte("bob"), 2) // "xyz", KeyProof, nil +// proof.Verify([]byte("bob"), val, root) // nil +// +// Proof of absence: +// +// _, proof, err = tree.GetVersionedWithProof([]byte("tom"), 2) // nil, KeyProof, nil +// proof.Verify([]byte("tom"), nil, root) // nil +// +// Now we delete an old version: +// +// tree.DeleteVersion(1) +// tree.VersionExists(1) // false +// tree.Get([]byte("alice")) // "xyz" +// tree.GetVersioned([]byte("alice"), 1) // nil +// +// Can't create a proof of absence for a version we no longer have: +// +// _, proof, err = tree.GetVersionedWithProof([]byte("tom"), 1) // nil, nil, error +// +package iavl diff --git a/glide.lock b/glide.lock index 137042e2e..087781d7c 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 012a45c86f12211f11600f0e690ce2c29f8c0f806af97b256c807268ccb20cfb -updated: 2017-09-01T13:35:18.837656965+02:00 +hash: d85321db3ba463c6bde8f76c65524d5bb8bcdbf43702d162b26b71a39d89a3b5 +updated: 2017-10-24T13:20:51.823688121+02:00 imports: - name: github.com/go-kit/kit version: d67bb4c202e3b91377d1079b110a6c9ce23ab2f8 @@ -40,18 +40,12 @@ imports: - leveldb/storage - leveldb/table - leveldb/util -- name: github.com/tendermint/abci - version: 01181721adca98ff9015ad8956a9e5cdc17d87d2 - subpackages: - - client - - server - - types - name: github.com/tendermint/go-wire - version: 5f88da3dbc1a72844e6dfaf274ce87f851d488eb + version: 99d2169a1e39c65983eacaa1da867d6f3218e1c9 subpackages: - data - name: github.com/tendermint/tmlibs - version: bfec1ff1cd7fda9f5b2d8b570e3bec163e5f9149 + version: d4c6a68e5871be2d3091f12627eaf3ee15b42ed2 subpackages: - common - db diff --git a/glide.yaml b/glide.yaml index 285f3ce4d..49b745798 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,11 +1,5 @@ package: github.com/tendermint/iavl import: -- package: github.com/tendermint/abci - version: develop - subpackages: - - client - - server - - types - package: github.com/tendermint/go-wire version: develop - package: github.com/tendermint/tmlibs diff --git a/iavl_node.go b/iavl_node.go deleted file mode 100644 index bee9296e0..000000000 --- a/iavl_node.go +++ /dev/null @@ -1,537 +0,0 @@ -package iavl - -import ( - "bytes" - "io" - - "golang.org/x/crypto/ripemd160" - - "github.com/tendermint/go-wire" - cmn "github.com/tendermint/tmlibs/common" -) - -// IAVLNode represents a node in an IAVLTree. -type IAVLNode struct { - key []byte - value []byte - height int8 - size int - hash []byte - leftHash []byte - leftNode *IAVLNode - rightHash []byte - rightNode *IAVLNode - persisted bool -} - -func NewIAVLNode(key []byte, value []byte) *IAVLNode { - return &IAVLNode{ - key: key, - value: value, - height: 0, - size: 1, - } -} - -// MakeIAVLNode constructs an *IAVLNode from an encoded byte slice. -// NOTE: The hash is not saved or set. The caller should set the hash afterwards. -func MakeIAVLNode(buf []byte) (node *IAVLNode, err error) { - node = &IAVLNode{} - - // Read node header. - - node.height = int8(buf[0]) - - n := 1 // Keeps track of bytes read. - buf = buf[n:] - - node.size, n, err = wire.GetVarint(buf) - if err != nil { - return nil, err - } - buf = buf[n:] - - node.key, n, err = wire.GetByteSlice(buf) - if err != nil { - return nil, err - } - buf = buf[n:] - - // Read node body. - - if node.isLeaf() { - node.value, n, err = wire.GetByteSlice(buf) - if err != nil { - return nil, err - } - } else { // Read children. - leftHash, n, err := wire.GetByteSlice(buf) - if err != nil { - return nil, err - } - buf = buf[n:] - - rightHash, _, err := wire.GetByteSlice(buf) - if err != nil { - return nil, err - } - node.leftHash = leftHash - node.rightHash = rightHash - } - return node, nil -} - -func (node *IAVLNode) _copy() *IAVLNode { - if node.isLeaf() { - cmn.PanicSanity("Why are you copying a value node?") - } - return &IAVLNode{ - key: node.key, - height: node.height, - size: node.size, - hash: nil, // Going to be mutated anyways. - leftHash: node.leftHash, - leftNode: node.leftNode, - rightHash: node.rightHash, - rightNode: node.rightNode, - persisted: false, // Going to be mutated, so it can't already be persisted. - } -} - -func (node *IAVLNode) isLeaf() bool { - return node.height == 0 -} - -// Check if the node has a descendant with the given key. -func (node *IAVLNode) has(t *IAVLTree, key []byte) (has bool) { - if bytes.Equal(node.key, key) { - return true - } - if node.isLeaf() { - return false - } - if bytes.Compare(key, node.key) < 0 { - return node.getLeftNode(t).has(t, key) - } else { - return node.getRightNode(t).has(t, key) - } -} - -func (node *IAVLNode) get(t *IAVLTree, key []byte) (index int, value []byte, exists bool) { - if node.isLeaf() { - switch bytes.Compare(node.key, key) { - case -1: - return 1, nil, false - case 1: - return 0, nil, false - default: - return 0, node.value, true - } - } - - if bytes.Compare(key, node.key) < 0 { - return node.getLeftNode(t).get(t, key) - } else { - rightNode := node.getRightNode(t) - index, value, exists = rightNode.get(t, key) - index += node.size - rightNode.size - return index, value, exists - } -} - -func (node *IAVLNode) getByIndex(t *IAVLTree, index int) (key []byte, value []byte) { - if node.isLeaf() { - if index == 0 { - return node.key, node.value - } else { - cmn.PanicSanity("getByIndex asked for invalid index") - return nil, nil - } - } else { - // TODO: could improve this by storing the - // sizes as well as left/right hash. - leftNode := node.getLeftNode(t) - if index < leftNode.size { - return leftNode.getByIndex(t, index) - } else { - return node.getRightNode(t).getByIndex(t, index-leftNode.size) - } - } -} - -// NOTE: sets hashes recursively -func (node *IAVLNode) hashWithCount() ([]byte, int) { - if node.hash != nil { - return node.hash, 0 - } - - hasher := ripemd160.New() - buf := new(bytes.Buffer) - _, hashCount, err := node.writeHashBytes(buf) - if err != nil { - cmn.PanicCrisis(err) - } - hasher.Write(buf.Bytes()) - node.hash = hasher.Sum(nil) - - return node.hash, hashCount + 1 -} - -// Writes the node's hash to the given io.Writer. -// This function has the side-effect of computing and setting the hashes of all descendant nodes. -func (node *IAVLNode) writeHashBytes(w io.Writer) (n int, hashCount int, err error) { - // height & size - wire.WriteInt8(node.height, w, &n, &err) - wire.WriteVarint(node.size, w, &n, &err) - // key is not written for inner nodes, unlike writeBytes - - if node.isLeaf() { - // key & value - wire.WriteByteSlice(node.key, w, &n, &err) - wire.WriteByteSlice(node.value, w, &n, &err) - } else { - // left - if node.leftNode != nil { - leftHash, leftCount := node.leftNode.hashWithCount() - node.leftHash = leftHash - hashCount += leftCount - } - if node.leftHash == nil { - cmn.PanicSanity("node.leftHash was nil in writeHashBytes") - } - wire.WriteByteSlice(node.leftHash, w, &n, &err) - - // right - if node.rightNode != nil { - rightHash, rightCount := node.rightNode.hashWithCount() - node.rightHash = rightHash - hashCount += rightCount - } - if node.rightHash == nil { - cmn.PanicSanity("node.rightHash was nil in writeHashBytes") - } - wire.WriteByteSlice(node.rightHash, w, &n, &err) - } - return -} - -// NOTE: clears leftNode/rigthNode recursively -// NOTE: sets hashes recursively -func (node *IAVLNode) save(t *IAVLTree) { - if node.hash == nil { - node.hash, _ = node.hashWithCount() - } - if node.persisted { - return - } - - // save children - if node.leftNode != nil { - node.leftNode.save(t) - node.leftNode = nil - } - if node.rightNode != nil { - node.rightNode.save(t) - node.rightNode = nil - } - - // save node - t.ndb.SaveNode(node) - return -} - -// NOTE: sets hashes recursively -func (node *IAVLNode) writeBytes(w io.Writer) (n int, err error) { - // node header - wire.WriteInt8(node.height, w, &n, &err) - wire.WriteVarint(node.size, w, &n, &err) - // key (unlike writeHashBytes, key is written for inner nodes) - wire.WriteByteSlice(node.key, w, &n, &err) - - if node.isLeaf() { - // value - wire.WriteByteSlice(node.value, w, &n, &err) - } else { - // left - if node.leftHash == nil { - cmn.PanicSanity("node.leftHash was nil in writeBytes") - } - wire.WriteByteSlice(node.leftHash, w, &n, &err) - // right - if node.rightHash == nil { - cmn.PanicSanity("node.rightHash was nil in writeBytes") - } - wire.WriteByteSlice(node.rightHash, w, &n, &err) - } - return -} - -func (node *IAVLNode) set(t *IAVLTree, key []byte, value []byte) (newSelf *IAVLNode, updated bool) { - if node.isLeaf() { - switch bytes.Compare(key, node.key) { - case -1: - return &IAVLNode{ - key: node.key, - height: 1, - size: 2, - leftNode: NewIAVLNode(key, value), - rightNode: node, - }, false - case 1: - return &IAVLNode{ - key: key, - height: 1, - size: 2, - leftNode: node, - rightNode: NewIAVLNode(key, value), - }, false - default: - removeOrphan(t, node) - return NewIAVLNode(key, value), true - } - } else { - removeOrphan(t, node) - node = node._copy() - if bytes.Compare(key, node.key) < 0 { - node.leftNode, updated = node.getLeftNode(t).set(t, key, value) - node.leftHash = nil // leftHash is yet unknown - } else { - node.rightNode, updated = node.getRightNode(t).set(t, key, value) - node.rightHash = nil // rightHash is yet unknown - } - if updated { - return node, updated - } else { - node.calcHeightAndSize(t) - return node.balance(t), updated - } - } -} - -// newHash/newNode: The new hash or node to replace node after remove. -// newKey: new leftmost leaf key for tree after successfully removing 'key' if changed. -// value: removed value. -func (node *IAVLNode) remove(t *IAVLTree, key []byte) ( - newHash []byte, newNode *IAVLNode, newKey []byte, value []byte, removed bool) { - if node.isLeaf() { - if bytes.Equal(key, node.key) { - removeOrphan(t, node) - return nil, nil, nil, node.value, true - } else { - return node.hash, node, nil, nil, false - } - } else { - if bytes.Compare(key, node.key) < 0 { - var newLeftHash []byte - var newLeftNode *IAVLNode - newLeftHash, newLeftNode, newKey, value, removed = node.getLeftNode(t).remove(t, key) - if !removed { - return node.hash, node, nil, value, false - } else if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed - return node.rightHash, node.rightNode, node.key, value, true - } - removeOrphan(t, node) - node = node._copy() - node.leftHash, node.leftNode = newLeftHash, newLeftNode - node.calcHeightAndSize(t) - node = node.balance(t) - return node.hash, node, newKey, value, true - } else { - var newRightHash []byte - var newRightNode *IAVLNode - newRightHash, newRightNode, newKey, value, removed = node.getRightNode(t).remove(t, key) - if !removed { - return node.hash, node, nil, value, false - } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed - return node.leftHash, node.leftNode, nil, value, true - } - removeOrphan(t, node) - node = node._copy() - node.rightHash, node.rightNode = newRightHash, newRightNode - if newKey != nil { - node.key = newKey - } - node.calcHeightAndSize(t) - node = node.balance(t) - return node.hash, node, nil, value, true - } - } -} - -func (node *IAVLNode) getLeftNode(t *IAVLTree) *IAVLNode { - if node.leftNode != nil { - return node.leftNode - } - return t.ndb.GetNode(node.leftHash) -} - -func (node *IAVLNode) getRightNode(t *IAVLTree) *IAVLNode { - if node.rightNode != nil { - return node.rightNode - } - return t.ndb.GetNode(node.rightHash) -} - -// NOTE: overwrites node -// TODO: optimize balance & rotate -func (node *IAVLNode) rotateRight(t *IAVLTree) *IAVLNode { - node = node._copy() - l := node.getLeftNode(t) - removeOrphan(t, l) - _l := l._copy() - - _lrHash, _lrCached := _l.rightHash, _l.rightNode - _l.rightHash, _l.rightNode = node.hash, node - node.leftHash, node.leftNode = _lrHash, _lrCached - - node.calcHeightAndSize(t) - _l.calcHeightAndSize(t) - - return _l -} - -// NOTE: overwrites node -// TODO: optimize balance & rotate -func (node *IAVLNode) rotateLeft(t *IAVLTree) *IAVLNode { - node = node._copy() - r := node.getRightNode(t) - removeOrphan(t, r) - _r := r._copy() - - _rlHash, _rlCached := _r.leftHash, _r.leftNode - _r.leftHash, _r.leftNode = node.hash, node - node.rightHash, node.rightNode = _rlHash, _rlCached - - node.calcHeightAndSize(t) - _r.calcHeightAndSize(t) - - return _r -} - -// NOTE: mutates height and size -func (node *IAVLNode) calcHeightAndSize(t *IAVLTree) { - node.height = maxInt8(node.getLeftNode(t).height, node.getRightNode(t).height) + 1 - node.size = node.getLeftNode(t).size + node.getRightNode(t).size -} - -func (node *IAVLNode) calcBalance(t *IAVLTree) int { - return int(node.getLeftNode(t).height) - int(node.getRightNode(t).height) -} - -// NOTE: assumes that node can be modified -// TODO: optimize balance & rotate -func (node *IAVLNode) balance(t *IAVLTree) (newSelf *IAVLNode) { - if node.persisted { - panic("Unexpected balance() call on persisted node") - } - balance := node.calcBalance(t) - if balance > 1 { - if node.getLeftNode(t).calcBalance(t) >= 0 { - // Left Left Case - return node.rotateRight(t) - } else { - // Left Right Case - // node = node._copy() - left := node.getLeftNode(t) - removeOrphan(t, left) - node.leftHash, node.leftNode = nil, left.rotateLeft(t) - //node.calcHeightAndSize() - return node.rotateRight(t) - } - } - if balance < -1 { - if node.getRightNode(t).calcBalance(t) <= 0 { - // Right Right Case - return node.rotateLeft(t) - } else { - // Right Left Case - // node = node._copy() - right := node.getRightNode(t) - removeOrphan(t, right) - node.rightHash, node.rightNode = nil, right.rotateRight(t) - //node.calcHeightAndSize() - return node.rotateLeft(t) - } - } - // Nothing changed - return node -} - -// traverse is a wrapper over traverseInRange when we want the whole tree -func (node *IAVLNode) traverse(t *IAVLTree, ascending bool, cb func(*IAVLNode) bool) bool { - return node.traverseInRange(t, nil, nil, ascending, false, cb) -} - -func (node *IAVLNode) traverseInRange(t *IAVLTree, start, end []byte, ascending bool, inclusive bool, cb func(*IAVLNode) bool) bool { - afterStart := start == nil || bytes.Compare(start, node.key) <= 0 - beforeEnd := end == nil || bytes.Compare(node.key, end) < 0 - if inclusive { - beforeEnd = end == nil || bytes.Compare(node.key, end) <= 0 - } - - stop := false - if afterStart && beforeEnd { - // IterateRange ignores this if not leaf - stop = cb(node) - } - if stop { - return stop - } - if node.isLeaf() { - return stop - } - - if ascending { - // check lower nodes, then higher - if afterStart { - stop = node.getLeftNode(t).traverseInRange(t, start, end, ascending, inclusive, cb) - } - if stop { - return stop - } - if beforeEnd { - stop = node.getRightNode(t).traverseInRange(t, start, end, ascending, inclusive, cb) - } - } else { - // check the higher nodes first - if beforeEnd { - stop = node.getRightNode(t).traverseInRange(t, start, end, ascending, inclusive, cb) - } - if stop { - return stop - } - if afterStart { - stop = node.getLeftNode(t).traverseInRange(t, start, end, ascending, inclusive, cb) - } - } - - return stop -} - -// Only used in testing... -func (node *IAVLNode) lmd(t *IAVLTree) *IAVLNode { - if node.isLeaf() { - return node - } - return node.getLeftNode(t).lmd(t) -} - -// Only used in testing... -func (node *IAVLNode) rmd(t *IAVLTree) *IAVLNode { - if node.isLeaf() { - return node - } - return node.getRightNode(t).rmd(t) -} - -//---------------------------------------- - -func removeOrphan(t *IAVLTree, node *IAVLNode) { - if !node.persisted { - return - } - if t.ndb == nil { - return - } - t.ndb.RemoveNode(t, node) -} diff --git a/iavl_nodedb.go b/iavl_nodedb.go deleted file mode 100644 index 6c2a0e9f5..000000000 --- a/iavl_nodedb.go +++ /dev/null @@ -1,140 +0,0 @@ -package iavl - -import ( - "bytes" - "container/list" - "sync" - - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" -) - -type nodeDB struct { - mtx sync.Mutex // Read/write lock. - cache map[string]*list.Element // Node cache. - cacheSize int // Node cache size limit in elements. - cacheQueue *list.List // LRU queue of cache elements. Used for deletion. - db dbm.DB // Persistent node storage. - batch dbm.Batch // Batched writing buffer. - orphans map[string]struct{} - orphansPrev map[string]struct{} -} - -func newNodeDB(cacheSize int, db dbm.DB) *nodeDB { - ndb := &nodeDB{ - cache: make(map[string]*list.Element), - cacheSize: cacheSize, - cacheQueue: list.New(), - db: db, - batch: db.NewBatch(), - orphans: make(map[string]struct{}), - orphansPrev: make(map[string]struct{}), - } - return ndb -} - -func (ndb *nodeDB) GetNode(hash []byte) *IAVLNode { - ndb.mtx.Lock() - defer ndb.mtx.Unlock() - - // Check the cache. - if elem, ok := ndb.cache[string(hash)]; ok { - // Already exists. Move to back of cacheQueue. - ndb.cacheQueue.MoveToBack(elem) - return elem.Value.(*IAVLNode) - } - - // Doesn't exist, load. - buf := ndb.db.Get(hash) - if len(buf) == 0 { - cmn.PanicSanity(cmn.Fmt("Value missing for key %X", hash)) - } - - node, err := MakeIAVLNode(buf) - if err != nil { - cmn.PanicCrisis(cmn.Fmt("Error reading IAVLNode. bytes: %X, error: %v", buf, err)) - } - - node.hash = hash - node.persisted = true - ndb.cacheNode(node) - - return node -} - -func (ndb *nodeDB) SaveNode(node *IAVLNode) { - ndb.mtx.Lock() - defer ndb.mtx.Unlock() - - if node.hash == nil { - cmn.PanicSanity("Expected to find node.hash, but none found.") - } - if node.persisted { - cmn.PanicSanity("Shouldn't be calling save on an already persisted node.") - } - - // Save node bytes to db - buf := new(bytes.Buffer) - if _, err := node.writeBytes(buf); err != nil { - cmn.PanicCrisis(err) - } - ndb.batch.Set(node.hash, buf.Bytes()) - node.persisted = true - ndb.cacheNode(node) - - // Re-creating the orphan, - // Do not garbage collect. - delete(ndb.orphans, string(node.hash)) - delete(ndb.orphansPrev, string(node.hash)) -} - -// Remove a node from cache and add it to the list of orphans, to be deleted -// on the next call to Commit. -func (ndb *nodeDB) RemoveNode(t *IAVLTree, node *IAVLNode) { - ndb.mtx.Lock() - defer ndb.mtx.Unlock() - - if node.hash == nil { - cmn.PanicSanity("Expected to find node.hash, but none found.") - } - if !node.persisted { - cmn.PanicSanity("Shouldn't be calling remove on a non-persisted node.") - } - - if elem, ok := ndb.cache[string(node.hash)]; ok { - ndb.cacheQueue.Remove(elem) - delete(ndb.cache, string(node.hash)) - } - ndb.orphans[string(node.hash)] = struct{}{} -} - -// Add a node to the cache and pop the least recently used node if we've -// reached the cache size limit. -func (ndb *nodeDB) cacheNode(node *IAVLNode) { - elem := ndb.cacheQueue.PushBack(node) - ndb.cache[string(node.hash)] = elem - - if ndb.cacheQueue.Len() > ndb.cacheSize { - oldest := ndb.cacheQueue.Front() - hash := ndb.cacheQueue.Remove(oldest).(*IAVLNode).hash - delete(ndb.cache, string(hash)) - } -} - -// Write to disk. Orphans are deleted here. -func (ndb *nodeDB) Commit() { - ndb.mtx.Lock() - defer ndb.mtx.Unlock() - - // Delete orphans from previous block - for orphanHashStr, _ := range ndb.orphansPrev { - ndb.batch.Delete([]byte(orphanHashStr)) - } - // Write saves & orphan deletes - ndb.batch.Write() - ndb.db.SetSync(nil, nil) - ndb.batch = ndb.db.NewBatch() - // Shift orphans - ndb.orphansPrev = ndb.orphans - ndb.orphans = make(map[string]struct{}) -} diff --git a/iavl_testutils_test.go b/iavl_testutils_test.go deleted file mode 100644 index 4523c74ec..000000000 --- a/iavl_testutils_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package iavl - -func dummyPathToKey(t *IAVLTree, key []byte) *PathToKey { - path, _, err := t.root.pathToKey(t, key) - if err != nil { - panic(err) - } - return path -} - -func dummyLeafNode(key, val []byte) IAVLProofLeafNode { - return IAVLProofLeafNode{key, val} -} diff --git a/iavl_tree.go b/iavl_tree.go deleted file mode 100644 index 76414d669..000000000 --- a/iavl_tree.go +++ /dev/null @@ -1,253 +0,0 @@ -package iavl - -import ( - "fmt" - "strings" - - cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" - - "github.com/pkg/errors" -) - -/* -Immutable AVL Tree (wraps the Node root) -This tree is not goroutine safe. -*/ -type IAVLTree struct { - root *IAVLNode - ndb *nodeDB -} - -// NewIAVLTree creates both im-memory and persistent instances -func NewIAVLTree(cacheSize int, db dbm.DB) *IAVLTree { - if db == nil { - // In-memory IAVLTree - return &IAVLTree{} - } else { - // Persistent IAVLTree - ndb := newNodeDB(cacheSize, db) - return &IAVLTree{ - ndb: ndb, - } - } -} - -// String returns a string representation of IAVLTree. -func (t *IAVLTree) String() string { - leaves := []string{} - t.Iterate(func(key []byte, val []byte) (stop bool) { - leaves = append(leaves, fmt.Sprintf("%x: %x", key, val)) - return false - }) - return "IAVLTree{" + strings.Join(leaves, ", ") + "}" -} - -// The returned tree and the original tree are goroutine independent. -// That is, they can each run in their own goroutine. -// However, upon Save(), any other trees that share a db will become -// outdated, as some nodes will become orphaned. -// Note that Save() clears leftNode and rightNode. Otherwise, -// two copies would not be goroutine independent. -func (t *IAVLTree) Copy() *IAVLTree { - if t.root == nil { - return &IAVLTree{ - root: nil, - ndb: t.ndb, - } - } - if t.ndb != nil && !t.root.persisted { - // Saving a tree finalizes all the nodes. - // It sets all the hashes recursively, - // clears all the leftNode/rightNode values recursively, - // and all the .persisted flags get set. - cmn.PanicSanity("It is unsafe to Copy() an unpersisted tree.") - } else if t.ndb == nil && t.root.hash == nil { - // An in-memory IAVLTree is finalized when the hashes are - // calculated. - t.root.hashWithCount() - } - return &IAVLTree{ - root: t.root, - ndb: t.ndb, - } -} - -func (t *IAVLTree) Size() int { - if t.root == nil { - return 0 - } - return t.root.size -} - -func (t *IAVLTree) Height() int8 { - if t.root == nil { - return 0 - } - return t.root.height -} - -func (t *IAVLTree) Has(key []byte) bool { - if t.root == nil { - return false - } - return t.root.has(t, key) -} - -func (t *IAVLTree) Set(key []byte, value []byte) (updated bool) { - if t.root == nil { - t.root = NewIAVLNode(key, value) - return false - } - t.root, updated = t.root.set(t, key, value) - return updated -} - -// BatchSet adds a Set to the current batch, will get handled atomically -func (t *IAVLTree) BatchSet(key []byte, value []byte) { - t.ndb.batch.Set(key, value) -} - -func (t *IAVLTree) Hash() []byte { - if t.root == nil { - return nil - } - hash, _ := t.root.hashWithCount() - return hash -} - -func (t *IAVLTree) HashWithCount() ([]byte, int) { - if t.root == nil { - return nil, 0 - } - return t.root.hashWithCount() -} - -func (t *IAVLTree) Save() []byte { - if t.root == nil { - return nil - } - if t.ndb != nil { - t.root.save(t) - t.ndb.Commit() - } - return t.root.hash -} - -// Sets the root node by reading from db. -// If the hash is empty, then sets root to nil. -func (t *IAVLTree) Load(hash []byte) { - if len(hash) == 0 { - t.root = nil - } else { - t.root = t.ndb.GetNode(hash) - } -} - -// Get returns the index and value of the specified key if it exists, or nil -// and the next index, if it doesn't. -func (t *IAVLTree) Get(key []byte) (index int, value []byte, exists bool) { - if t.root == nil { - return 0, nil, false - } - return t.root.get(t, key) -} - -func (t *IAVLTree) GetByIndex(index int) (key []byte, value []byte) { - if t.root == nil { - return nil, nil - } - return t.root.getByIndex(t, index) -} - -// GetWithProof gets the value under the key if it exists, or returns nil. -// A proof of existence or absence is returned alongside the value. -func (t *IAVLTree) GetWithProof(key []byte) ([]byte, KeyProof, error) { - value, eproof, err := t.getWithProof(key) - if err == nil { - return value, eproof, nil - } - - aproof, err := t.keyAbsentProof(key) - if err == nil { - return nil, aproof, nil - } - return nil, nil, errors.Wrap(err, "could not construct any proof") -} - -// GetRangeWithProof gets key/value pairs within the specified range and limit. To specify a descending -// range, swap the start and end keys. -// -// Returns a list of keys, a list of values and a proof. -func (t *IAVLTree) GetRangeWithProof(startKey []byte, endKey []byte, limit int) ([][]byte, [][]byte, *KeyRangeProof, error) { - return t.getRangeWithProof(startKey, endKey, limit) -} - -// GetFirstInRangeWithProof gets the first key/value pair in the specified range, with a proof. -func (t *IAVLTree) GetFirstInRangeWithProof(startKey, endKey []byte) ([]byte, []byte, *KeyFirstInRangeProof, error) { - return t.getFirstInRangeWithProof(startKey, endKey) -} - -// GetLastInRangeWithProof gets the last key/value pair in the specified range, with a proof. -func (t *IAVLTree) GetLastInRangeWithProof(startKey, endKey []byte) ([]byte, []byte, *KeyLastInRangeProof, error) { - return t.getLastInRangeWithProof(startKey, endKey) -} - -func (t *IAVLTree) Remove(key []byte) (value []byte, removed bool) { - if t.root == nil { - return nil, false - } - newRootHash, newRoot, _, value, removed := t.root.remove(t, key) - if !removed { - return nil, false - } - if newRoot == nil && newRootHash != nil { - t.root = t.ndb.GetNode(newRootHash) - } else { - t.root = newRoot - } - return value, true -} - -func (t *IAVLTree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { - if t.root == nil { - return false - } - return t.root.traverse(t, true, func(node *IAVLNode) bool { - if node.height == 0 { - return fn(node.key, node.value) - } else { - return false - } - }) -} - -// IterateRange makes a callback for all nodes with key between start and end non-inclusive. -// If either are nil, then it is open on that side (nil, nil is the same as Iterate) -func (t *IAVLTree) IterateRange(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { - if t.root == nil { - return false - } - return t.root.traverseInRange(t, start, end, ascending, false, func(node *IAVLNode) bool { - if node.height == 0 { - return fn(node.key, node.value) - } else { - return false - } - }) -} - -// IterateRangeInclusive makes a callback for all nodes with key between start and end inclusive. -// If either are nil, then it is open on that side (nil, nil is the same as Iterate) -func (t *IAVLTree) IterateRangeInclusive(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { - if t.root == nil { - return false - } - return t.root.traverseInRange(t, start, end, ascending, true, func(node *IAVLNode) bool { - if node.height == 0 { - return fn(node.key, node.value) - } else { - return false - } - }) -} diff --git a/iavl_tree_dump.go b/iavl_tree_dump.go deleted file mode 100644 index 54f1e6d19..000000000 --- a/iavl_tree_dump.go +++ /dev/null @@ -1,150 +0,0 @@ -package iavl - -import ( - "bytes" - "fmt" - "io" - "os" - - wire "github.com/tendermint/go-wire" -) - -type Formatter func(in []byte) (out string) - -type KeyValueMapping struct { - Key Formatter - Value Formatter -} - -// Flip back and forth between ascii and hex. -func mixedDisplay(value []byte) string { - - var buffer bytes.Buffer - var last []byte - - ascii := true - for i := 0; i < len(value); i++ { - if value[i] < 32 || value[i] > 126 { - if ascii && len(last) > 0 { - // only if there are 6 or more chars - if len(last) > 5 { - buffer.WriteString(fmt.Sprintf("%s", last)) - last = nil - } - ascii = false - } - } - last = append(last, value[i]) - } - if ascii { - buffer.WriteString(fmt.Sprintf("%s", last)) - } else { - buffer.WriteString(fmt.Sprintf("%X", last)) - } - return buffer.String() -} - -// This is merkleeyes state, that it is writing to a specific key -type state struct { - Hash []byte - Height uint64 -} - -// Try to interpet as merkleeyes state -func stateMapping(value []byte) string { - var s state - err := wire.ReadBinaryBytes(value, &s) - if err != nil || s.Height > 500 { - return mixedDisplay(value) - } - return fmt.Sprintf("Height:%d, [%X]", s.Height, s.Hash) -} - -// This is basecoin accounts, that it is writing to a specific key -type account struct { - PubKey []byte - Sequence int - Balance []coin -} - -type wrapper struct { - bytes []byte -} - -type coin struct { - Denom string - Amount int64 -} - -// Perhaps this is an IAVL tree node? -func nodeMapping(node *IAVLNode) string { - - formattedKey := mixedDisplay(node.key) - - var formattedValue string - var acc account - - err := wire.ReadBinaryBytes(node.value, &acc) - if err != nil { - formattedValue = mixedDisplay(node.value) - } else { - formattedValue = fmt.Sprintf("%v", acc) - } - - if node.height == 0 { - return fmt.Sprintf(" LeafNode[height: %d, size %d, key: %s, value: %s]", - node.height, node.size, formattedKey, formattedValue) - } else { - return fmt.Sprintf("InnerNode[height: %d, size %d, key: %s, leftHash: %X, rightHash: %X]", - node.height, node.size, formattedKey, node.leftHash, node.rightHash) - } -} - -// Try everything and see what sticks... -func overallMapping(value []byte) (str string) { - // underneath make node, wire can throw a panic - defer func() { - if recover() != nil { - str = fmt.Sprintf("%X", value) - return - } - }() - - // test to see if this is a node - node, err := MakeIAVLNode(value) - - if err == nil && node.height < 100 && node.key != nil { - return nodeMapping(node) - } - - // Unknown value type - return stateMapping(value) -} - -// Dump everything from the database to stdout. -func (t *IAVLTree) Dump(verbose bool, mapping *KeyValueMapping) { - t.Fdump(os.Stdout, verbose, mapping) -} - -// Fdump serializes the entire database to the provided io.Writer. -func (t *IAVLTree) Fdump(w io.Writer, verbose bool, mapping *KeyValueMapping) { - if verbose && t.root == nil { - fmt.Fprintf(w, "No root loaded into memory\n") - } - - if mapping == nil { - mapping = &KeyValueMapping{Key: mixedDisplay, Value: overallMapping} - } - - if verbose { - stats := t.ndb.db.Stats() - for key, value := range stats { - fmt.Fprintf(w, "%s:\n\t%s\n", key, value) - } - } - - iter := t.ndb.db.Iterator() - for iter.Next() { - fmt.Fprintf(w, "%s: %s\n", mapping.Key(iter.Key()), mapping.Value(iter.Value())) - } -} diff --git a/iavl_tree_dump_test.go b/iavl_tree_dump_test.go deleted file mode 100644 index 30e7ba778..000000000 --- a/iavl_tree_dump_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package iavl - -import ( - "io/ioutil" - "testing" - - "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/db" -) - -func TestIAVLTreeFdump(t *testing.T) { - t.Skipf("Tree dump and DB code seem buggy so this test always crashes. See https://github.com/tendermint/tmlibs/issues/36") - db := db.NewDB("test", db.MemDBBackendStr, "") - tree := NewIAVLTree(100000, db) - for i := 0; i < 1000000; i++ { - tree.Set(i2b(int(common.RandInt32())), nil) - if i > 990000 && i%1000 == 999 { - tree.Save() - } - } - tree.Save() - - // insert lots of info and store the bytes - for i := 0; i < 200; i++ { - key, value := common.RandStr(20), common.RandStr(200) - tree.Set([]byte(key), []byte(value)) - } - - tree.Fdump(ioutil.Discard, true, nil) -} diff --git a/node.go b/node.go new file mode 100644 index 000000000..a785feba9 --- /dev/null +++ b/node.go @@ -0,0 +1,554 @@ +package iavl + +import ( + "bytes" + "fmt" + "io" + + "golang.org/x/crypto/ripemd160" + + "github.com/tendermint/go-wire" + cmn "github.com/tendermint/tmlibs/common" +) + +// Node represents a node in a Tree. +type Node struct { + key []byte + value []byte + version uint64 + height int8 + size int + hash []byte + leftHash []byte + leftNode *Node + rightHash []byte + rightNode *Node + persisted bool +} + +// NewNode returns a new node from a key and value. +func NewNode(key []byte, value []byte) *Node { + return &Node{ + key: key, + value: value, + height: 0, + size: 1, + version: 0, + } +} + +// MakeNode constructs an *Node from an encoded byte slice. +// +// The new node doesn't have its hash saved or set. The caller must set it +// afterwards. +func MakeNode(buf []byte) (node *Node, err error) { + node = &Node{} + + // Read node header. + + node.height = int8(buf[0]) + + n := 1 // Keeps track of bytes read. + buf = buf[n:] + + node.size, n, err = wire.GetVarint(buf) + if err != nil { + return nil, err + } + buf = buf[n:] + + node.key, n, err = wire.GetByteSlice(buf) + if err != nil { + return nil, err + } + buf = buf[n:] + + node.version = wire.GetUint64(buf) + buf = buf[8:] + + // Read node body. + + if node.isLeaf() { + node.value, _, err = wire.GetByteSlice(buf) + if err != nil { + return nil, err + } + } else { // Read children. + leftHash, n, err := wire.GetByteSlice(buf) + if err != nil { + return nil, err + } + buf = buf[n:] + + rightHash, _, err := wire.GetByteSlice(buf) + if err != nil { + return nil, err + } + node.leftHash = leftHash + node.rightHash = rightHash + } + return node, nil +} + +// String returns a string representation of the node. +func (node *Node) String() string { + if len(node.hash) == 0 { + return "" + } else { + return fmt.Sprintf("%x", node.hash) + } +} + +// clone creates a shallow copy of a node with its hash set to nil. +func (node *Node) clone() *Node { + if node.isLeaf() { + cmn.PanicSanity("Attempt to copy a leaf node") + } + return &Node{ + key: node.key, + height: node.height, + version: node.version, + size: node.size, + hash: nil, + leftHash: node.leftHash, + leftNode: node.leftNode, + rightHash: node.rightHash, + rightNode: node.rightNode, + persisted: false, + } +} + +func (node *Node) isLeaf() bool { + return node.height == 0 +} + +// Check if the node has a descendant with the given key. +func (node *Node) has(t *Tree, key []byte) (has bool) { + if bytes.Equal(node.key, key) { + return true + } + if node.isLeaf() { + return false + } + if bytes.Compare(key, node.key) < 0 { + return node.getLeftNode(t).has(t, key) + } else { + return node.getRightNode(t).has(t, key) + } +} + +// Get a key under the node. +func (node *Node) get(t *Tree, key []byte) (index int, value []byte) { + if node.isLeaf() { + switch bytes.Compare(node.key, key) { + case -1: + return 1, nil + case 1: + return 0, nil + default: + return 0, node.value + } + } + + if bytes.Compare(key, node.key) < 0 { + return node.getLeftNode(t).get(t, key) + } else { + rightNode := node.getRightNode(t) + index, value = rightNode.get(t, key) + index += node.size - rightNode.size + return index, value + } +} + +func (node *Node) getByIndex(t *Tree, index int) (key []byte, value []byte) { + if node.isLeaf() { + if index == 0 { + return node.key, node.value + } else { + return nil, nil + } + } else { + // TODO: could improve this by storing the + // sizes as well as left/right hash. + leftNode := node.getLeftNode(t) + if index < leftNode.size { + return leftNode.getByIndex(t, index) + } else { + return node.getRightNode(t).getByIndex(t, index-leftNode.size) + } + } +} + +// Computes the hash of the node without computing its descendants. Must be +// called on nodes which have descendant node hashes already computed. +func (node *Node) _hash() []byte { + if node.hash != nil { + return node.hash + } + + hasher := ripemd160.New() + buf := new(bytes.Buffer) + if _, err := node.writeHashBytes(buf); err != nil { + cmn.PanicCrisis(err) + } + hasher.Write(buf.Bytes()) + node.hash = hasher.Sum(nil) + + return node.hash +} + +// Hash the node and its descendants recursively. This usually mutates all +// descendant nodes. Returns the node hash and number of nodes hashed. +func (node *Node) hashWithCount() ([]byte, int) { + if node.hash != nil { + return node.hash, 0 + } + + hasher := ripemd160.New() + buf := new(bytes.Buffer) + _, hashCount, err := node.writeHashBytesRecursively(buf) + if err != nil { + cmn.PanicCrisis(err) + } + hasher.Write(buf.Bytes()) + node.hash = hasher.Sum(nil) + + return node.hash, hashCount + 1 +} + +// Writes the node's hash to the given io.Writer. This function expects +// child hashes to be already set. +func (node *Node) writeHashBytes(w io.Writer) (n int, err error) { + wire.WriteInt8(node.height, w, &n, &err) + wire.WriteVarint(node.size, w, &n, &err) + + // Key is not written for inner nodes, unlike writeBytes. + + if node.isLeaf() { + wire.WriteByteSlice(node.key, w, &n, &err) + wire.WriteByteSlice(node.value, w, &n, &err) + wire.WriteUint64(node.version, w, &n, &err) + } else { + if node.leftHash == nil || node.rightHash == nil { + cmn.PanicSanity("Found an empty child hash") + } + wire.WriteByteSlice(node.leftHash, w, &n, &err) + wire.WriteByteSlice(node.rightHash, w, &n, &err) + } + return +} + +// Writes the node's hash to the given io.Writer. +// This function has the side-effect of calling hashWithCount. +func (node *Node) writeHashBytesRecursively(w io.Writer) (n int, hashCount int, err error) { + if node.leftNode != nil { + leftHash, leftCount := node.leftNode.hashWithCount() + node.leftHash = leftHash + hashCount += leftCount + } + if node.rightNode != nil { + rightHash, rightCount := node.rightNode.hashWithCount() + node.rightHash = rightHash + hashCount += rightCount + } + n, err = node.writeHashBytes(w) + + return +} + +// Writes the node as a serialized byte slice to the supplied io.Writer. +func (node *Node) writeBytes(w io.Writer) (n int, err error) { + wire.WriteInt8(node.height, w, &n, &err) + wire.WriteVarint(node.size, w, &n, &err) + + // Unlike writeHashBytes, key is written for inner nodes. + wire.WriteByteSlice(node.key, w, &n, &err) + wire.WriteUint64(node.version, w, &n, &err) + + if node.isLeaf() { + wire.WriteByteSlice(node.value, w, &n, &err) + } else { + if node.leftHash == nil { + cmn.PanicSanity("node.leftHash was nil in writeBytes") + } + wire.WriteByteSlice(node.leftHash, w, &n, &err) + + if node.rightHash == nil { + cmn.PanicSanity("node.rightHash was nil in writeBytes") + } + wire.WriteByteSlice(node.rightHash, w, &n, &err) + } + return +} + +func (node *Node) set(t *Tree, key []byte, value []byte) ( + newSelf *Node, updated bool, orphaned []*Node, +) { + if node.isLeaf() { + switch bytes.Compare(key, node.key) { + case -1: + return &Node{ + key: node.key, + height: 1, + size: 2, + leftNode: NewNode(key, value), + rightNode: node, + }, false, []*Node{} + case 1: + return &Node{ + key: key, + height: 1, + size: 2, + leftNode: node, + rightNode: NewNode(key, value), + }, false, []*Node{} + default: + return NewNode(key, value), true, []*Node{node} + } + } else { + orphaned = append(orphaned, node) + node = node.clone() + + if bytes.Compare(key, node.key) < 0 { + var leftOrphaned []*Node + node.leftNode, updated, leftOrphaned = node.getLeftNode(t).set(t, key, value) + node.leftHash = nil // leftHash is yet unknown + orphaned = append(orphaned, leftOrphaned...) + } else { + var rightOrphaned []*Node + node.rightNode, updated, rightOrphaned = node.getRightNode(t).set(t, key, value) + node.rightHash = nil // rightHash is yet unknown + orphaned = append(orphaned, rightOrphaned...) + } + + if updated { + return node, updated, orphaned + } else { + node.calcHeightAndSize(t) + newNode, balanceOrphaned := node.balance(t) + return newNode, updated, append(orphaned, balanceOrphaned...) + } + } +} + +// newHash/newNode: The new hash or node to replace node after remove. +// newKey: new leftmost leaf key for tree after successfully removing 'key' if changed. +// value: removed value. +func (node *Node) remove(t *Tree, key []byte) ( + newHash []byte, newNode *Node, newKey []byte, value []byte, orphaned []*Node, +) { + if node.isLeaf() { + if bytes.Equal(key, node.key) { + return nil, nil, nil, node.value, []*Node{node} + } + return node.hash, node, nil, nil, orphaned + } + + if bytes.Compare(key, node.key) < 0 { + var newLeftHash []byte + var newLeftNode *Node + + newLeftHash, newLeftNode, newKey, value, orphaned = + node.getLeftNode(t).remove(t, key) + + if len(orphaned) == 0 { + return node.hash, node, nil, value, orphaned + } else if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed + return node.rightHash, node.rightNode, node.key, value, orphaned + } + orphaned = append(orphaned, node) + + newNode := node.clone() + newNode.leftHash, newNode.leftNode = newLeftHash, newLeftNode + newNode.calcHeightAndSize(t) + newNode, balanceOrphaned := newNode.balance(t) + + return newNode.hash, newNode, newKey, value, append(orphaned, balanceOrphaned...) + } else { + var newRightHash []byte + var newRightNode *Node + + newRightHash, newRightNode, newKey, value, orphaned = + node.getRightNode(t).remove(t, key) + + if len(orphaned) == 0 { + return node.hash, node, nil, value, orphaned + } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed + return node.leftHash, node.leftNode, nil, value, orphaned + } + orphaned = append(orphaned, node) + + newNode := node.clone() + newNode.rightHash, newNode.rightNode = newRightHash, newRightNode + if newKey != nil { + newNode.key = newKey + } + newNode.calcHeightAndSize(t) + newNode, balanceOrphaned := newNode.balance(t) + + return newNode.hash, newNode, nil, value, append(orphaned, balanceOrphaned...) + } +} + +func (node *Node) getLeftNode(t *Tree) *Node { + if node.leftNode != nil { + return node.leftNode + } + return t.ndb.GetNode(node.leftHash) +} + +func (node *Node) getRightNode(t *Tree) *Node { + if node.rightNode != nil { + return node.rightNode + } + return t.ndb.GetNode(node.rightHash) +} + +// Rotate right and return the new node and orphan. +func (node *Node) rotateRight(t *Tree) (newNode *Node, orphan *Node) { + // TODO: optimize balance & rotate. + node = node.clone() + l := node.getLeftNode(t) + _l := l.clone() + + _lrHash, _lrCached := _l.rightHash, _l.rightNode + _l.rightHash, _l.rightNode = node.hash, node + node.leftHash, node.leftNode = _lrHash, _lrCached + + node.calcHeightAndSize(t) + _l.calcHeightAndSize(t) + + return _l, l +} + +// Rotate left and return the new node and orphan. +func (node *Node) rotateLeft(t *Tree) (newNode *Node, orphan *Node) { + // TODO: optimize balance & rotate. + node = node.clone() + r := node.getRightNode(t) + _r := r.clone() + + _rlHash, _rlCached := _r.leftHash, _r.leftNode + _r.leftHash, _r.leftNode = node.hash, node + node.rightHash, node.rightNode = _rlHash, _rlCached + + node.calcHeightAndSize(t) + _r.calcHeightAndSize(t) + + return _r, r +} + +// NOTE: mutates height and size +func (node *Node) calcHeightAndSize(t *Tree) { + node.height = maxInt8(node.getLeftNode(t).height, node.getRightNode(t).height) + 1 + node.size = node.getLeftNode(t).size + node.getRightNode(t).size +} + +func (node *Node) calcBalance(t *Tree) int { + return int(node.getLeftNode(t).height) - int(node.getRightNode(t).height) +} + +// NOTE: assumes that node can be modified +// TODO: optimize balance & rotate +func (node *Node) balance(t *Tree) (newSelf *Node, orphaned []*Node) { + if node.persisted { + panic("Unexpected balance() call on persisted node") + } + balance := node.calcBalance(t) + + if balance > 1 { + if node.getLeftNode(t).calcBalance(t) >= 0 { + // Left Left Case + newNode, orphaned := node.rotateRight(t) + return newNode, []*Node{orphaned} + } else { + // Left Right Case + var leftOrphaned *Node + + left := node.getLeftNode(t) + node.leftHash = nil + node.leftNode, leftOrphaned = left.rotateLeft(t) + newNode, rightOrphaned := node.rotateRight(t) + + return newNode, []*Node{left, leftOrphaned, rightOrphaned} + } + } + if balance < -1 { + if node.getRightNode(t).calcBalance(t) <= 0 { + // Right Right Case + newNode, orphaned := node.rotateLeft(t) + return newNode, []*Node{orphaned} + } else { + // Right Left Case + var rightOrphaned *Node + + right := node.getRightNode(t) + node.rightHash = nil + node.rightNode, rightOrphaned = right.rotateRight(t) + newNode, leftOrphaned := node.rotateLeft(t) + + return newNode, []*Node{right, leftOrphaned, rightOrphaned} + } + } + // Nothing changed + return node, []*Node{} +} + +// traverse is a wrapper over traverseInRange when we want the whole tree +func (node *Node) traverse(t *Tree, ascending bool, cb func(*Node) bool) bool { + return node.traverseInRange(t, nil, nil, ascending, false, cb) +} + +func (node *Node) traverseInRange(t *Tree, start, end []byte, ascending bool, inclusive bool, cb func(*Node) bool) bool { + afterStart := start == nil || bytes.Compare(start, node.key) <= 0 + beforeEnd := end == nil || bytes.Compare(node.key, end) < 0 + if inclusive { + beforeEnd = end == nil || bytes.Compare(node.key, end) <= 0 + } + + stop := false + if afterStart && beforeEnd { + // IterateRange ignores this if not leaf + stop = cb(node) + } + if stop { + return stop + } + if node.isLeaf() { + return stop + } + + if ascending { + // check lower nodes, then higher + if afterStart { + stop = node.getLeftNode(t).traverseInRange(t, start, end, ascending, inclusive, cb) + } + if stop { + return stop + } + if beforeEnd { + stop = node.getRightNode(t).traverseInRange(t, start, end, ascending, inclusive, cb) + } + } else { + // check the higher nodes first + if beforeEnd { + stop = node.getRightNode(t).traverseInRange(t, start, end, ascending, inclusive, cb) + } + if stop { + return stop + } + if afterStart { + stop = node.getLeftNode(t).traverseInRange(t, start, end, ascending, inclusive, cb) + } + } + + return stop +} + +// Only used in testing... +func (node *Node) lmd(t *Tree) *Node { + if node.isLeaf() { + return node + } + return node.getLeftNode(t).lmd(t) +} diff --git a/nodedb.go b/nodedb.go new file mode 100644 index 000000000..79ac4b3fc --- /dev/null +++ b/nodedb.go @@ -0,0 +1,504 @@ +package iavl + +import ( + "bytes" + "container/list" + "errors" + "fmt" + "sort" + "sync" + + cmn "github.com/tendermint/tmlibs/common" + dbm "github.com/tendermint/tmlibs/db" +) + +var ( + // All node keys are prefixed with this. This ensures no collision is + // possible with the other keys, and makes them easier to traverse. + nodesPrefix = "n/" + nodesKeyFmt = "n/%x" + + // Orphans are keyed in the database by their expected lifetime. + // The first number represents the *last* version at which the orphan needs + // to exist, while the second number represents the *earliest* version at + // which it is expected to exist - which starts out by being the version + // of the node being orphaned. + orphansPrefix = "o/" + orphansPrefixFmt = "o/%d/" // o// + orphansKeyFmt = "o/%d/%d/%x" // o/// + + // These keys are used for the orphan reverse-lookups by node hash. + orphansIndexPrefix = "O/" + orphansIndexKeyFmt = "O/%x" + + // r/ + rootsPrefix = "r/" + rootsPrefixFmt = "r/%d" +) + +type nodeDB struct { + mtx sync.Mutex // Read/write lock. + db dbm.DB // Persistent node storage. + batch dbm.Batch // Batched writing buffer. + + versionCache map[uint64][]byte // Cache of tree (root) versions. + latestVersion uint64 // Latest root version. + + nodeCache map[string]*list.Element // Node cache. + nodeCacheSize int // Node cache size limit in elements. + nodeCacheQueue *list.List // LRU queue of cache elements. Used for deletion. +} + +func newNodeDB(cacheSize int, db dbm.DB) *nodeDB { + ndb := &nodeDB{ + nodeCache: make(map[string]*list.Element), + nodeCacheSize: cacheSize, + nodeCacheQueue: list.New(), + db: db, + batch: db.NewBatch(), + versionCache: map[uint64][]byte{}, + } + return ndb +} + +// GetNode gets a node from cache or disk. If it is an inner node, it does not +// load its children. +func (ndb *nodeDB) GetNode(hash []byte) *Node { + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + + // Check the cache. + if elem, ok := ndb.nodeCache[string(hash)]; ok { + // Already exists. Move to back of nodeCacheQueue. + ndb.nodeCacheQueue.MoveToBack(elem) + return elem.Value.(*Node) + } + + // Doesn't exist, load. + buf := ndb.db.Get(ndb.nodeKey(hash)) + if buf == nil { + cmn.PanicSanity(cmn.Fmt("Value missing for key %x", hash)) + } + + node, err := MakeNode(buf) + if err != nil { + cmn.PanicCrisis(cmn.Fmt("Error reading Node. bytes: %x, error: %v", buf, err)) + } + + node.hash = hash + node.persisted = true + ndb.cacheNode(node) + + return node +} + +// SaveNode saves a node to disk. +func (ndb *nodeDB) SaveNode(node *Node) { + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + + if node.hash == nil { + cmn.PanicSanity("Expected to find node.hash, but none found.") + } + if node.persisted { + cmn.PanicSanity("Shouldn't be calling save on an already persisted node.") + } + + // Save node bytes to db. + buf := new(bytes.Buffer) + if _, err := node.writeBytes(buf); err != nil { + cmn.PanicCrisis(err) + } + ndb.batch.Set(ndb.nodeKey(node.hash), buf.Bytes()) + + node.persisted = true + ndb.cacheNode(node) +} + +// Has checks if a hash exists in the database. +func (ndb *nodeDB) Has(hash []byte) bool { + key := ndb.nodeKey(hash) + + if ldb, ok := ndb.db.(*dbm.GoLevelDB); ok { + exists, err := ldb.DB().Has(key, nil) + if err != nil { + cmn.PanicSanity("Got error from leveldb: " + err.Error()) + } + return exists + } + return ndb.db.Get(key) != nil +} + +// SaveBranch saves the given node and all of its descendants. For each node +// about to be saved, the supplied callback is called and the returned node is +// is saved. You may pass nil as the callback as a pass-through. +// +// Note that this function clears leftNode/rigthNode recursively and calls +// hashWithCount on the given node. +func (ndb *nodeDB) SaveBranch(node *Node, cb func(*Node)) []byte { + if node.persisted { + return node.hash + } + + if node.leftNode != nil { + node.leftHash = ndb.SaveBranch(node.leftNode, cb) + } + if node.rightNode != nil { + node.rightHash = ndb.SaveBranch(node.rightNode, cb) + } + + if cb != nil { + cb(node) + } + + node._hash() + ndb.SaveNode(node) + + node.leftNode = nil + node.rightNode = nil + + return node.hash +} + +// DeleteVersion deletes a tree version from disk. +func (ndb *nodeDB) DeleteVersion(version uint64) { + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + + ndb.deleteOrphans(version) + ndb.deleteRoot(version) +} + +// Unorphan deletes the orphan entry from disk, but not the node it points to. +func (ndb *nodeDB) Unorphan(hash []byte) { + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + + indexKey := ndb.orphanIndexKey(hash) + + if orphansKey := ndb.db.Get(indexKey); len(orphansKey) > 0 { + ndb.batch.Delete(orphansKey) + ndb.batch.Delete(indexKey) + } +} + +// Saves orphaned nodes to disk under a special prefix. +func (ndb *nodeDB) SaveOrphans(version uint64, orphans map[string]uint64) { + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + + toVersion := ndb.getPreviousVersion(version) + + for hash, fromVersion := range orphans { + ndb.saveOrphan([]byte(hash), fromVersion, toVersion) + } +} + +// Saves a single orphan to disk. +func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion uint64) { + if fromVersion > toVersion { + cmn.PanicSanity("Orphan expires before it comes alive") + } + key := ndb.orphanKey(fromVersion, toVersion, hash) + ndb.batch.Set(key, hash) + + // Set reverse-lookup index. + indexKey := ndb.orphanIndexKey(hash) + ndb.batch.Set(indexKey, key) +} + +// deleteOrphans deletes orphaned nodes from disk, and the associated orphan +// entries. +func (ndb *nodeDB) deleteOrphans(version uint64) { + // Will be zero if there is no previous version. + predecessor := ndb.getPreviousVersion(version) + + // Traverse orphans with a lifetime ending at the version specified. + ndb.traverseOrphansVersion(version, func(key, hash []byte) { + var fromVersion, toVersion uint64 + + // See comment on `orphansKeyFmt`. Note that here, `version` and + // `toVersion` are always equal. + fmt.Sscanf(string(key), orphansKeyFmt, &toVersion, &fromVersion) + + // Delete orphan key and reverse-lookup key. + ndb.batch.Delete(key) + ndb.batch.Delete(ndb.orphanIndexKey(hash)) + + // If there is no predecessor, or the predecessor is earlier than the + // beginning of the lifetime (ie: negative lifetime), or the lifetime + // spans a single version and that version is the one being deleted, we + // can delete the orphan. Otherwise, we shorten its lifetime, by + // moving its endpoint to the previous version. + if predecessor < fromVersion || fromVersion == toVersion { + ndb.batch.Delete(ndb.nodeKey(hash)) + ndb.uncacheNode(hash) + } else { + ndb.saveOrphan(hash, fromVersion, predecessor) + } + }) +} + +func (ndb *nodeDB) nodeKey(hash []byte) []byte { + return []byte(fmt.Sprintf(nodesKeyFmt, hash)) +} + +func (ndb *nodeDB) orphanIndexKey(hash []byte) []byte { + return []byte(fmt.Sprintf(orphansIndexKeyFmt, hash)) +} + +func (ndb *nodeDB) orphanKey(fromVersion, toVersion uint64, hash []byte) []byte { + return []byte(fmt.Sprintf(orphansKeyFmt, toVersion, fromVersion, hash)) +} + +func (ndb *nodeDB) rootKey(version uint64) []byte { + return []byte(fmt.Sprintf(rootsPrefixFmt, version)) +} + +func (ndb *nodeDB) getLatestVersion() uint64 { + if ndb.latestVersion == 0 { + ndb.getVersions() + } + return ndb.latestVersion +} + +func (ndb *nodeDB) getVersions() map[uint64][]byte { + if len(ndb.versionCache) == 0 { + ndb.traversePrefix([]byte(rootsPrefix), func(k, hash []byte) { + var version uint64 + fmt.Sscanf(string(k), rootsPrefixFmt, &version) + ndb.cacheVersion(version, hash) + }) + } + return ndb.versionCache +} + +func (ndb *nodeDB) cacheVersion(version uint64, hash []byte) { + ndb.versionCache[version] = hash + + if version > ndb.getLatestVersion() { + ndb.latestVersion = version + } +} + +func (ndb *nodeDB) getPreviousVersion(version uint64) uint64 { + var result uint64 + for v := range ndb.getVersions() { + if v < version && v > result { + result = v + } + } + return result +} + +// deleteRoot deletes the root entry from disk, but not the node it points to. +func (ndb *nodeDB) deleteRoot(version uint64) { + key := ndb.rootKey(version) + ndb.batch.Delete(key) + + delete(ndb.versionCache, version) + + if version == ndb.getLatestVersion() { + cmn.PanicSanity("Tried to delete latest version") + } +} + +func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { + ndb.traversePrefix([]byte(orphansPrefix), fn) +} + +// Traverse orphans ending at a certain version. +func (ndb *nodeDB) traverseOrphansVersion(version uint64, fn func(k, v []byte)) { + prefix := fmt.Sprintf(orphansPrefixFmt, version) + ndb.traversePrefix([]byte(prefix), fn) +} + +// Traverse all keys. +func (ndb *nodeDB) traverse(fn func(key, value []byte)) { + it := ndb.db.Iterator() + defer it.Release() + + for it.Next() { + fn(it.Key(), it.Value()) + } + if err := it.Error(); err != nil { + cmn.PanicSanity(err.Error()) + } +} + +// Traverse all keys with a certain prefix. +func (ndb *nodeDB) traversePrefix(prefix []byte, fn func(k, v []byte)) { + it := ndb.db.IteratorPrefix(prefix) + defer it.Release() + + for it.Next() { + fn(it.Key(), it.Value()) + } + if err := it.Error(); err != nil { + cmn.PanicSanity(err.Error()) + } +} + +func (ndb *nodeDB) uncacheNode(hash []byte) { + if elem, ok := ndb.nodeCache[string(hash)]; ok { + ndb.nodeCacheQueue.Remove(elem) + delete(ndb.nodeCache, string(hash)) + } +} + +// Add a node to the cache and pop the least recently used node if we've +// reached the cache size limit. +func (ndb *nodeDB) cacheNode(node *Node) { + elem := ndb.nodeCacheQueue.PushBack(node) + ndb.nodeCache[string(node.hash)] = elem + + if ndb.nodeCacheQueue.Len() > ndb.nodeCacheSize { + oldest := ndb.nodeCacheQueue.Front() + hash := ndb.nodeCacheQueue.Remove(oldest).(*Node).hash + delete(ndb.nodeCache, string(hash)) + } +} + +// Write to disk. +func (ndb *nodeDB) Commit() { + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + + ndb.batch.Write() + ndb.batch = ndb.db.NewBatch() +} + +func (ndb *nodeDB) getRoots() (map[uint64][]byte, error) { + roots := map[uint64][]byte{} + + ndb.traversePrefix([]byte(rootsPrefix), func(k, v []byte) { + var version uint64 + fmt.Sscanf(string(k), rootsPrefixFmt, &version) + roots[version] = v + }) + return roots, nil +} + +// SaveRoot creates an entry on disk for the given root, so that it can be +// loaded later. +func (ndb *nodeDB) SaveRoot(root *Node, version uint64) error { + ndb.mtx.Lock() + defer ndb.mtx.Unlock() + + if len(root.hash) == 0 { + cmn.PanicSanity("Hash should not be empty") + } + if version <= ndb.getLatestVersion() { + return errors.New("can't save root with lower or equal version than latest") + } + + // Note that we don't use the version attribute of the root. This is + // because we might be saving an old root at a new version in the case + // where the tree wasn't modified between versions. + key := ndb.rootKey(version) + ndb.batch.Set(key, root.hash) + ndb.cacheVersion(version, root.hash) + + return nil +} + +////////////////// Utility and test functions ///////////////////////////////// + +func (ndb *nodeDB) leafNodes() []*Node { + leaves := []*Node{} + + ndb.traverseNodes(func(hash []byte, node *Node) { + if node.isLeaf() { + leaves = append(leaves, node) + } + }) + return leaves +} + +func (ndb *nodeDB) nodes() []*Node { + nodes := []*Node{} + + ndb.traverseNodes(func(hash []byte, node *Node) { + nodes = append(nodes, node) + }) + return nodes +} + +func (ndb *nodeDB) orphans() [][]byte { + orphans := [][]byte{} + + ndb.traverseOrphans(func(k, v []byte) { + orphans = append(orphans, v) + }) + return orphans +} + +func (ndb *nodeDB) roots() map[uint64][]byte { + roots, _ := ndb.getRoots() + return roots +} + +func (ndb *nodeDB) size() int { + it := ndb.db.Iterator() + size := 0 + + for it.Next() { + size++ + } + return size +} + +func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node)) { + nodes := []*Node{} + + ndb.traversePrefix([]byte(nodesPrefix), func(key, value []byte) { + node, err := MakeNode(value) + if err != nil { + cmn.PanicSanity("Couldn't decode node from database") + } + fmt.Sscanf(string(key), nodesKeyFmt, &node.hash) + nodes = append(nodes, node) + }) + + sort.Slice(nodes, func(i, j int) bool { + return bytes.Compare(nodes[i].key, nodes[j].key) < 0 + }) + + for _, n := range nodes { + fn(n.hash, n) + } +} + +func (ndb *nodeDB) String() string { + var str string + index := 0 + + ndb.traversePrefix([]byte(rootsPrefix), func(key, value []byte) { + str += fmt.Sprintf("%s: %x\n", string(key), value) + }) + str += "\n" + + ndb.traverseOrphans(func(key, value []byte) { + str += fmt.Sprintf("%s: %x\n", string(key), value) + }) + str += "\n" + + ndb.traversePrefix([]byte(orphansIndexPrefix), func(key, value []byte) { + str += fmt.Sprintf("%s: %s\n", string(key), value) + }) + str += "\n" + + ndb.traverseNodes(func(hash []byte, node *Node) { + if len(hash) == 0 { + str += fmt.Sprintf("\n") + } else if node == nil { + str += fmt.Sprintf("%s%40x: \n", nodesPrefix, hash) + } else if node.value == nil && node.height > 0 { + str += fmt.Sprintf("%s%40x: %s %-16s h=%d version=%d\n", nodesPrefix, hash, node.key, "", node.height, node.version) + } else { + str += fmt.Sprintf("%s%40x: %s = %-16s h=%d version=%d\n", nodesPrefix, hash, node.key, node.value, node.height, node.version) + } + index++ + }) + return "-" + "\n" + str + "-" +} diff --git a/orphaning_tree.go b/orphaning_tree.go new file mode 100644 index 000000000..432cb4b0e --- /dev/null +++ b/orphaning_tree.go @@ -0,0 +1,87 @@ +package iavl + +import ( + cmn "github.com/tendermint/tmlibs/common" +) + +// orphaningTree is a tree which keeps track of orphaned nodes. +type orphaningTree struct { + *Tree + + // A map of orphan hash to orphan version. + // The version stored here is the one at which the orphan's lifetime + // begins. + orphans map[string]uint64 +} + +// newOrphaningTree creates a new orphaning tree from the given *Tree. +func newOrphaningTree(t *Tree) *orphaningTree { + return &orphaningTree{ + Tree: t, + orphans: map[string]uint64{}, + } +} + +// Set a key on the underlying tree while storing the orphaned nodes. +func (tree *orphaningTree) Set(key, value []byte) bool { + orphaned, updated := tree.Tree.set(key, value) + tree.addOrphans(orphaned) + return updated +} + +// Remove a key from the underlying tree while storing the orphaned nodes. +func (tree *orphaningTree) Remove(key []byte) ([]byte, bool) { + val, orphaned, removed := tree.Tree.remove(key) + tree.addOrphans(orphaned) + return val, removed +} + +// Unorphan undoes the orphaning of a node, removing the orphan entry on disk +// if necessary. +func (tree *orphaningTree) unorphan(hash []byte) { + tree.deleteOrphan(hash) + tree.ndb.Unorphan(hash) +} + +// Save the underlying Tree. Saves orphans too. +func (tree *orphaningTree) SaveVersion(version uint64) { + // Save the current tree at the given version. For each saved node, we + // delete any existing orphan entries in the previous trees. + // This is necessary because sometimes tree re-balancing causes nodes to be + // incorrectly marked as orphaned, since tree patterns after a re-balance + // may mirror previous tree patterns, with matching hashes. + tree.ndb.SaveBranch(tree.root, func(node *Node) { + // The node version is set here since it isn't known until we save. + // Note that we only want to set the version for inner nodes the first + // time, as they represent the beginning of the lifetime of that node. + // So unless it's a leaf node, we only update version when it's 0. + if node.version == 0 || node.isLeaf() { + node.version = version + } + tree.unorphan(node._hash()) + }) + tree.ndb.SaveOrphans(version, tree.orphans) +} + +// Add orphans to the orphan list. Doesn't write to disk. +func (tree *orphaningTree) addOrphans(orphans []*Node) { + for _, node := range orphans { + if !node.persisted { + // We don't need to orphan nodes that were never persisted. + continue + } + if len(node.hash) == 0 { + cmn.PanicSanity("Expected to find node hash, but was empty") + } + tree.orphans[string(node.hash)] = node.version + } +} + +// Delete an orphan from the orphan list. Doesn't write to disk. +func (tree *orphaningTree) deleteOrphan(hash []byte) (version uint64, deleted bool) { + if version, ok := tree.orphans[string(hash)]; ok { + delete(tree.orphans, string(hash)) + return version, true + } + return 0, false +} diff --git a/iavl_path.go b/path.go similarity index 82% rename from iavl_path.go rename to path.go index 3adb04d1c..3351fbefc 100644 --- a/iavl_path.go +++ b/path.go @@ -10,7 +10,7 @@ import ( // Note that the nodes are ordered such that the last one is closest // to the root of the tree. type PathToKey struct { - InnerNodes []IAVLProofInnerNode `json:"inner_nodes"` + InnerNodes []proofInnerNode `json:"inner_nodes"` } func (p *PathToKey) String() string { @@ -23,13 +23,13 @@ func (p *PathToKey) String() string { // verify check that the leafNode's hash matches the path's LeafHash and that // the root is the merkle hash of all the inner nodes. -func (p *PathToKey) verify(leafNode IAVLProofLeafNode, root []byte) error { - leafHash := leafNode.Hash() +func (p *PathToKey) verify(leafNode proofLeafNode, root []byte) error { + hash := leafNode.Hash() for _, branch := range p.InnerNodes { - leafHash = branch.Hash(leafHash) + hash = branch.Hash(hash) } - if !bytes.Equal(root, leafHash) { - return ErrInvalidProof() + if !bytes.Equal(root, hash) { + return errors.WithStack(ErrInvalidProof) } return nil } @@ -86,35 +86,35 @@ func (p *PathToKey) isLeftAdjacentTo(p2 *PathToKey) bool { } // PathWithNode is a path to a key which includes the leaf node at that key. -type PathWithNode struct { - Path *PathToKey `json:"path"` - Node IAVLProofLeafNode `json:"node"` +type pathWithNode struct { + Path *PathToKey `json:"path"` + Node proofLeafNode `json:"node"` } -func (p *PathWithNode) verify(root []byte) error { +func (p *pathWithNode) verify(root []byte) error { return p.Path.verify(p.Node, root) } // verifyPaths verifies the left and right paths individually, and makes sure // the ordering is such that left < startKey <= endKey < right. -func verifyPaths(left, right *PathWithNode, startKey, endKey, root []byte) error { +func verifyPaths(left, right *pathWithNode, startKey, endKey, root []byte) error { if bytes.Compare(startKey, endKey) == 1 { return ErrInvalidInputs } if left != nil { if err := left.verify(root); err != nil { - return ErrInvalidProof() + return err } if !left.Node.isLesserThan(startKey) { - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } } if right != nil { if err := right.verify(root); err != nil { - return ErrInvalidProof() + return err } if !right.Node.isGreaterThan(endKey) { - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } } return nil @@ -140,7 +140,7 @@ func verifyNoMissingKeys(paths []*PathToKey) error { // Checks that with the given left and right paths, no keys can exist in between. // Supports nil paths to signify out-of-range. -func verifyKeyAbsence(left, right *PathWithNode) error { +func verifyKeyAbsence(left, right *pathWithNode) error { if left != nil && left.Path.isRightmost() { // Range starts outside of the right boundary. return nil @@ -152,5 +152,5 @@ func verifyKeyAbsence(left, right *PathWithNode) error { // Range is between two existing keys. return nil } - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } diff --git a/iavl_proof.go b/proof.go similarity index 60% rename from iavl_proof.go rename to proof.go index ae8e8b2ca..5c87eae83 100644 --- a/iavl_proof.go +++ b/proof.go @@ -9,13 +9,12 @@ import ( "github.com/tendermint/go-wire" "github.com/tendermint/go-wire/data" - . "github.com/tendermint/tmlibs/common" + cmn "github.com/tendermint/tmlibs/common" ) -const proofLimit = 1 << 16 // 64 KB - var ( - errInvalidProof = fmt.Errorf("invalid proof") + // ErrInvalidProof is returned by Verify when a proof cannot be validated. + ErrInvalidProof = fmt.Errorf("invalid proof") // ErrInvalidInputs is returned when the inputs passed to the function are invalid. ErrInvalidInputs = fmt.Errorf("invalid inputs") @@ -27,28 +26,25 @@ var ( ErrNilRoot = fmt.Errorf("tree root is nil") ) -// ErrInvalidProof is returned by Verify when a proof cannot be validated. -func ErrInvalidProof() error { - return errors.WithStack(errInvalidProof) -} - -type IAVLProofInnerNode struct { +type proofInnerNode struct { Height int8 Size int Left []byte Right []byte } -func (n *IAVLProofInnerNode) String() string { - return fmt.Sprintf("IAVLProofInnerNode[height=%d, %x / %x]", n.Height, n.Left, n.Right) +func (n *proofInnerNode) String() string { + return fmt.Sprintf("proofInnerNode[height=%d, %x / %x]", n.Height, n.Left, n.Right) } -func (branch IAVLProofInnerNode) Hash(childHash []byte) []byte { +func (branch proofInnerNode) Hash(childHash []byte) []byte { hasher := ripemd160.New() buf := new(bytes.Buffer) n, err := int(0), error(nil) + wire.WriteInt8(branch.Height, buf, &n, &err) wire.WriteVarint(branch.Size, buf, &n, &err) + if len(branch.Left) == 0 { wire.WriteByteSlice(childHash, buf, &n, &err) wire.WriteByteSlice(branch.Right, buf, &n, &err) @@ -57,88 +53,92 @@ func (branch IAVLProofInnerNode) Hash(childHash []byte) []byte { wire.WriteByteSlice(childHash, buf, &n, &err) } if err != nil { - PanicCrisis(Fmt("Failed to hash IAVLProofInnerNode: %v", err)) + cmn.PanicCrisis(cmn.Fmt("Failed to hash proofInnerNode: %v", err)) } - // fmt.Printf("InnerNode hash bytes: %X\n", buf.Bytes()) hasher.Write(buf.Bytes()) + return hasher.Sum(nil) } -type IAVLProofLeafNode struct { +type proofLeafNode struct { KeyBytes data.Bytes `json:"key"` ValueBytes data.Bytes `json:"value"` + Version uint64 `json:"version"` } -func (leaf IAVLProofLeafNode) Hash() []byte { +func (leaf proofLeafNode) Hash() []byte { hasher := ripemd160.New() buf := new(bytes.Buffer) n, err := int(0), error(nil) + wire.WriteInt8(0, buf, &n, &err) wire.WriteVarint(1, buf, &n, &err) wire.WriteByteSlice(leaf.KeyBytes, buf, &n, &err) wire.WriteByteSlice(leaf.ValueBytes, buf, &n, &err) + wire.WriteUint64(leaf.Version, buf, &n, &err) + if err != nil { - PanicCrisis(Fmt("Failed to hash IAVLProofLeafNode: %v", err)) + cmn.PanicCrisis(cmn.Fmt("Failed to hash proofLeafNode: %v", err)) } - // fmt.Printf("LeafNode hash bytes: %X\n", buf.Bytes()) hasher.Write(buf.Bytes()) + return hasher.Sum(nil) } -func (leaf IAVLProofLeafNode) isLesserThan(key []byte) bool { +func (leaf proofLeafNode) isLesserThan(key []byte) bool { return bytes.Compare(leaf.KeyBytes, key) == -1 } -func (leaf IAVLProofLeafNode) isGreaterThan(key []byte) bool { +func (leaf proofLeafNode) isGreaterThan(key []byte) bool { return bytes.Compare(leaf.KeyBytes, key) == 1 } -func (node *IAVLNode) pathToKey(t *IAVLTree, key []byte) (*PathToKey, []byte, error) { +func (node *Node) pathToKey(t *Tree, key []byte) (*PathToKey, *Node, error) { path := &PathToKey{} val, err := node._pathToKey(t, key, path) return path, val, err } -func (node *IAVLNode) _pathToKey(t *IAVLTree, key []byte, path *PathToKey) ([]byte, error) { +func (node *Node) _pathToKey(t *Tree, key []byte, path *PathToKey) (*Node, error) { if node.height == 0 { - if bytes.Compare(node.key, key) == 0 { - return node.value, nil + if bytes.Equal(node.key, key) { + return node, nil } return nil, errors.New("key does not exist") } if bytes.Compare(key, node.key) < 0 { - if value, err := node.getLeftNode(t)._pathToKey(t, key, path); err != nil { + if n, err := node.getLeftNode(t)._pathToKey(t, key, path); err != nil { return nil, err } else { - branch := IAVLProofInnerNode{ + branch := proofInnerNode{ Height: node.height, Size: node.size, Left: nil, Right: node.getRightNode(t).hash, } path.InnerNodes = append(path.InnerNodes, branch) - return value, nil + return n, nil } } - if value, err := node.getRightNode(t)._pathToKey(t, key, path); err != nil { + if n, err := node.getRightNode(t)._pathToKey(t, key, path); err != nil { return nil, err } else { - branch := IAVLProofInnerNode{ + branch := proofInnerNode{ Height: node.height, Size: node.size, Left: node.getLeftNode(t).hash, Right: nil, } path.InnerNodes = append(path.InnerNodes, branch) - return value, nil + return n, nil } } -func (t *IAVLTree) constructKeyAbsentProof(key []byte, proof *KeyAbsentProof) error { +func (t *Tree) constructKeyAbsentProof(key []byte, proof *KeyAbsentProof) error { // Get the index of the first key greater than the requested key, if the key doesn't exist. - idx, _, exists := t.Get(key) - if exists { + idx, val := t.Get(key) + if val != nil { return errors.Errorf("couldn't construct non-existence proof: key 0x%x exists", key) } @@ -158,30 +158,30 @@ func (t *IAVLTree) constructKeyAbsentProof(key []byte, proof *KeyAbsentProof) er } if lkey != nil { - path, _, _ := t.root.pathToKey(t, lkey) - proof.Left = &PathWithNode{ + path, node, _ := t.root.pathToKey(t, lkey) + proof.Left = &pathWithNode{ Path: path, - Node: IAVLProofLeafNode{KeyBytes: lkey, ValueBytes: lval}, + Node: proofLeafNode{lkey, lval, node.version}, } } if rkey != nil { - path, _, _ := t.root.pathToKey(t, rkey) - proof.Right = &PathWithNode{ + path, node, _ := t.root.pathToKey(t, rkey) + proof.Right = &pathWithNode{ Path: path, - Node: IAVLProofLeafNode{KeyBytes: rkey, ValueBytes: rval}, + Node: proofLeafNode{rkey, rval, node.version}, } } return nil } -func (t *IAVLTree) getWithProof(key []byte) (value []byte, proof *KeyExistsProof, err error) { +func (t *Tree) getWithProof(key []byte) (value []byte, proof *KeyExistsProof, err error) { if t.root == nil { - return nil, nil, ErrNilRoot + return nil, nil, errors.WithStack(ErrNilRoot) } t.root.hashWithCount() // Ensure that all hashes are calculated. - path, value, err := t.root.pathToKey(t, key) + path, node, err := t.root.pathToKey(t, key) if err != nil { return nil, nil, errors.Wrap(err, "could not construct path to key") } @@ -189,17 +189,20 @@ func (t *IAVLTree) getWithProof(key []byte) (value []byte, proof *KeyExistsProof proof = &KeyExistsProof{ RootHash: t.root.hash, PathToKey: path, + Version: node.version, } - return value, proof, nil + return node.value, proof, nil } -func (t *IAVLTree) keyAbsentProof(key []byte) (*KeyAbsentProof, error) { +func (t *Tree) keyAbsentProof(key []byte) (*KeyAbsentProof, error) { if t.root == nil { - return nil, ErrNilRoot + return nil, errors.WithStack(ErrNilRoot) } t.root.hashWithCount() // Ensure that all hashes are calculated. + proof := &KeyAbsentProof{ RootHash: t.root.hash, + Version: t.root.version, } if err := t.constructKeyAbsentProof(key, proof); err != nil { return nil, errors.Wrap(err, "could not construct proof of non-existence") diff --git a/iavl_proof_key.go b/proof_key.go similarity index 63% rename from iavl_proof_key.go rename to proof_key.go index e52f09026..a09c93fc3 100644 --- a/iavl_proof_key.go +++ b/proof_key.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" + "github.com/pkg/errors" "github.com/tendermint/go-wire" "github.com/tendermint/go-wire/data" ) @@ -16,11 +17,16 @@ type KeyProof interface { // Root returns the root hash of the proof. Root() []byte + + // Serialize itself + Bytes() []byte } // KeyExistsProof represents a proof of existence of a single key. type KeyExistsProof struct { - RootHash data.Bytes `json:"root_hash"` + RootHash data.Bytes `json:"root_hash"` + Version uint64 `json:"version"` + *PathToKey `json:"path"` } @@ -31,28 +37,33 @@ func (proof *KeyExistsProof) Root() []byte { // Verify verifies the proof is valid and returns an error if it isn't. func (proof *KeyExistsProof) Verify(key []byte, value []byte, root []byte) error { if !bytes.Equal(proof.RootHash, root) { - return ErrInvalidRoot + return errors.WithStack(ErrInvalidRoot) } if key == nil || value == nil { - return ErrInvalidInputs + return errors.WithStack(ErrInvalidInputs) } - return proof.PathToKey.verify(IAVLProofLeafNode{key, value}, root) + return proof.PathToKey.verify(proofLeafNode{key, value, proof.Version}, root) +} + +// Bytes returns a go-wire binary serialization +func (proof *KeyExistsProof) Bytes() []byte { + return wire.BinaryBytes(proof) } // ReadKeyExistsProof will deserialize a KeyExistsProof from bytes. func ReadKeyExistsProof(data []byte) (*KeyExistsProof, error) { - // TODO: make go-wire never panic - n, err := int(0), error(nil) - proof := wire.ReadBinary(&KeyExistsProof{}, bytes.NewBuffer(data), proofLimit, &n, &err).(*KeyExistsProof) + proof := new(KeyExistsProof) + err := wire.ReadBinaryBytes(data, &proof) return proof, err } // KeyAbsentProof represents a proof of the absence of a single key. type KeyAbsentProof struct { RootHash data.Bytes `json:"root_hash"` + Version uint64 `json:"version"` - Left *PathWithNode `json:"left"` - Right *PathWithNode `json:"right"` + Left *pathWithNode `json:"left"` + Right *pathWithNode `json:"right"` } func (proof *KeyAbsentProof) Root() []byte { @@ -66,14 +77,14 @@ func (p *KeyAbsentProof) String() string { // Verify verifies the proof is valid and returns an error if it isn't. func (proof *KeyAbsentProof) Verify(key, value []byte, root []byte) error { if !bytes.Equal(proof.RootHash, root) { - return ErrInvalidRoot + return errors.WithStack(ErrInvalidRoot) } if key == nil || value != nil { return ErrInvalidInputs } if proof.Left == nil && proof.Right == nil { - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } if err := verifyPaths(proof.Left, proof.Right, key, key, root); err != nil { return err @@ -81,3 +92,15 @@ func (proof *KeyAbsentProof) Verify(key, value []byte, root []byte) error { return verifyKeyAbsence(proof.Left, proof.Right) } + +// Bytes returns a go-wire binary serialization +func (proof *KeyAbsentProof) Bytes() []byte { + return wire.BinaryBytes(proof) +} + +// ReadKeyAbsentProof will deserialize a KeyAbsentProof from bytes. +func ReadKeyAbsentProof(data []byte) (*KeyAbsentProof, error) { + proof := new(KeyAbsentProof) + err := wire.ReadBinaryBytes(data, &proof) + return proof, err +} diff --git a/proof_key_test.go b/proof_key_test.go new file mode 100644 index 000000000..027add234 --- /dev/null +++ b/proof_key_test.go @@ -0,0 +1,45 @@ +package iavl + +import ( + "testing" + + "github.com/stretchr/testify/require" + + cmn "github.com/tendermint/tmlibs/common" +) + +func TestSerializeProofs(t *testing.T) { + require := require.New(t) + + tree := NewTree(0, nil) + for _, ikey := range []byte{0x17, 0x42, 0x99} { + key := []byte{ikey} + tree.Set(key, cmn.RandBytes(8)) + } + root := tree.Hash() + + // test with key exists + key := []byte{0x17} + val, proof, err := tree.GetWithProof(key) + require.Nil(err, "%+v", err) + require.NotNil(val) + bin := proof.Bytes() + eproof, err := ReadKeyExistsProof(bin) + require.Nil(err, "%+v", err) + require.NoError(eproof.Verify(key, val, root)) + _, err = ReadKeyAbsentProof(bin) + require.NotNil(err) + + // test with key absent + key = []byte{0x38} + val, proof, err = tree.GetWithProof(key) + require.Nil(err, "%+v", err) + require.Nil(val) + bin = proof.Bytes() + // I think this is ugly it works this way, but without type-bytes nothing we can do :( + // eproof, err = ReadKeyExistsProof(bin) + // require.NotNil(err) + aproof, err := ReadKeyAbsentProof(bin) + require.Nil(err, "%+v", err) + require.NoError(aproof.Verify(key, val, root)) +} diff --git a/iavl_proof_range.go b/proof_range.go similarity index 85% rename from iavl_proof_range.go rename to proof_range.go index 963290d3a..07cac7424 100644 --- a/iavl_proof_range.go +++ b/proof_range.go @@ -17,8 +17,8 @@ type KeyInRangeProof interface { type KeyFirstInRangeProof struct { KeyExistsProof `json:"key_proof"` - Left *PathWithNode `json:"left"` - Right *PathWithNode `json:"right"` + Left *pathWithNode `json:"left"` + Right *pathWithNode `json:"right"` } // String returns a string representation of the proof. @@ -35,7 +35,7 @@ func (proof *KeyFirstInRangeProof) Verify(startKey, endKey, key, value []byte, r } } if proof.Left == nil && proof.Right == nil && proof.PathToKey == nil { - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } if err := verifyPaths(proof.Left, proof.Right, startKey, endKey, root); err != nil { return err @@ -64,7 +64,7 @@ func (proof *KeyFirstInRangeProof) Verify(startKey, endKey, key, value []byte, r if proof.Left != nil && proof.Left.Path.isLeftAdjacentTo(proof.PathToKey) { return nil } - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } /////////////////////////////////////////////////////////////////////////////// @@ -73,8 +73,8 @@ func (proof *KeyFirstInRangeProof) Verify(startKey, endKey, key, value []byte, r type KeyLastInRangeProof struct { KeyExistsProof `json:"key_proof"` - Left *PathWithNode `json:"left"` - Right *PathWithNode `json:"right"` + Left *pathWithNode `json:"left"` + Right *pathWithNode `json:"right"` } // String returns a string representation of the proof. @@ -89,7 +89,7 @@ func (proof *KeyLastInRangeProof) Verify(startKey, endKey, key, value []byte, ro return ErrInvalidInputs } if proof.Left == nil && proof.Right == nil && proof.PathToKey == nil { - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } if err := verifyPaths(proof.Left, proof.Right, startKey, endKey, root); err != nil { return err @@ -113,7 +113,7 @@ func (proof *KeyLastInRangeProof) Verify(startKey, endKey, key, value []byte, ro return nil } - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } /////////////////////////////////////////////////////////////////////////////// @@ -121,10 +121,11 @@ func (proof *KeyLastInRangeProof) Verify(startKey, endKey, key, value []byte, ro // KeyRangeProof is proof that a range of keys does or does not exist. type KeyRangeProof struct { RootHash data.Bytes `json:"root_hash"` + Version uint64 `json:"version"` PathToKeys []*PathToKey `json:"paths"` - Left *PathWithNode `json:"left"` - Right *PathWithNode `json:"right"` + Left *pathWithNode `json:"left"` + Right *pathWithNode `json:"right"` } // Verify that a range proof is valid. @@ -176,7 +177,7 @@ func (proof *KeyRangeProof) Verify( // If we've reached this point, it means our range isn't empty, and we have // a list of keys. for i, path := range proof.PathToKeys { - leafNode := IAVLProofLeafNode{KeyBytes: keys[i], ValueBytes: values[i]} + leafNode := proofLeafNode{KeyBytes: keys[i], ValueBytes: values[i]} if err := path.verify(leafNode, root); err != nil { return errors.WithStack(err) } @@ -191,13 +192,13 @@ func (proof *KeyRangeProof) Verify( if proof.Left == nil && !bytes.Equal(startKey, keys[0]) && !proof.PathToKeys[0].isLeftmost() { - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } if proof.Right == nil && !bytes.Equal(endKey, keys[len(keys)-1]) && !proof.PathToKeys[len(proof.PathToKeys)-1].isRightmost() { - return ErrInvalidProof() + return errors.WithStack(ErrInvalidProof) } return nil } @@ -222,7 +223,7 @@ func (proof *KeyRangeProof) paths() []*PathToKey { /////////////////////////////////////////////////////////////////////////////// -func (t *IAVLTree) getRangeWithProof(keyStart, keyEnd []byte, limit int) ( +func (t *Tree) getRangeWithProof(keyStart, keyEnd []byte, limit int) ( keys, values [][]byte, rangeProof *KeyRangeProof, err error, ) { if t.root == nil { @@ -293,12 +294,12 @@ func (t *IAVLTree) getRangeWithProof(keyStart, keyEnd []byte, limit int) ( if needsLeft { // Find index of first key to the left, and include proof if it isn't the // leftmost key. - if idx, _, _ := t.Get(rangeStart); idx > 0 { + if idx, _ := t.Get(rangeStart); idx > 0 { lkey, lval := t.GetByIndex(idx - 1) path, _, _ := t.root.pathToKey(t, lkey) - rangeProof.Left = &PathWithNode{ + rangeProof.Left = &pathWithNode{ Path: path, - Node: IAVLProofLeafNode{KeyBytes: lkey, ValueBytes: lval}, + Node: proofLeafNode{KeyBytes: lkey, ValueBytes: lval}, } } } @@ -309,12 +310,12 @@ func (t *IAVLTree) getRangeWithProof(keyStart, keyEnd []byte, limit int) ( if needsRight { // Find index of first key to the right, and include proof if it isn't the // rightmost key. - if idx, _, _ := t.Get(rangeEnd); idx <= t.Size()-1 { + if idx, _ := t.Get(rangeEnd); idx <= t.Size()-1 { rkey, rval := t.GetByIndex(idx) path, _, _ := t.root.pathToKey(t, rkey) - rangeProof.Right = &PathWithNode{ + rangeProof.Right = &pathWithNode{ Path: path, - Node: IAVLProofLeafNode{KeyBytes: rkey, ValueBytes: rval}, + Node: proofLeafNode{KeyBytes: rkey, ValueBytes: rval}, } } } @@ -322,7 +323,7 @@ func (t *IAVLTree) getRangeWithProof(keyStart, keyEnd []byte, limit int) ( return keys, values, rangeProof, nil } -func (t *IAVLTree) getFirstInRangeWithProof(keyStart, keyEnd []byte) ( +func (t *Tree) getFirstInRangeWithProof(keyStart, keyEnd []byte) ( key, value []byte, proof *KeyFirstInRangeProof, err error, ) { if t.root == nil { @@ -343,27 +344,31 @@ func (t *IAVLTree) getFirstInRangeWithProof(keyStart, keyEnd []byte) ( } if !bytes.Equal(key, keyStart) { - if idx, _, _ := t.Get(keyStart); idx-1 >= 0 && idx-1 <= t.Size()-1 { + if idx, _ := t.Get(keyStart); idx-1 >= 0 && idx-1 <= t.Size()-1 { k, v := t.GetByIndex(idx - 1) - proof.Left = &PathWithNode{} - proof.Left.Path, _, _ = t.root.pathToKey(t, k) - proof.Left.Node = IAVLProofLeafNode{k, v} + path, node, _ := t.root.pathToKey(t, k) + proof.Left = &pathWithNode{ + Path: path, + Node: proofLeafNode{k, v, node.version}, + } } } if !bytes.Equal(key, keyEnd) { - if idx, _, exists := t.Get(keyEnd); idx <= t.Size()-1 && !exists { + if idx, val := t.Get(keyEnd); idx <= t.Size()-1 && val == nil { k, v := t.GetByIndex(idx) - proof.Right = &PathWithNode{} - proof.Right.Path, _, _ = t.root.pathToKey(t, k) - proof.Right.Node = IAVLProofLeafNode{KeyBytes: k, ValueBytes: v} + path, node, _ := t.root.pathToKey(t, k) + proof.Right = &pathWithNode{ + Path: path, + Node: proofLeafNode{k, v, node.version}, + } } } return key, value, proof, nil } -func (t *IAVLTree) getLastInRangeWithProof(keyStart, keyEnd []byte) ( +func (t *Tree) getLastInRangeWithProof(keyStart, keyEnd []byte) ( key, value []byte, proof *KeyLastInRangeProof, err error, ) { if t.root == nil { @@ -385,20 +390,24 @@ func (t *IAVLTree) getLastInRangeWithProof(keyStart, keyEnd []byte) ( } if !bytes.Equal(key, keyEnd) { - if idx, _, _ := t.Get(keyEnd); idx <= t.Size()-1 { + if idx, _ := t.Get(keyEnd); idx <= t.Size()-1 { k, v := t.GetByIndex(idx) - proof.Right = &PathWithNode{} - proof.Right.Path, _, _ = t.root.pathToKey(t, k) - proof.Right.Node = IAVLProofLeafNode{KeyBytes: k, ValueBytes: v} + path, node, _ := t.root.pathToKey(t, k) + proof.Right = &pathWithNode{ + Path: path, + Node: proofLeafNode{k, v, node.version}, + } } } if !bytes.Equal(key, keyStart) { - if idx, _, _ := t.Get(keyStart); idx-1 >= 0 && idx-1 <= t.Size()-1 { + if idx, _ := t.Get(keyStart); idx-1 >= 0 && idx-1 <= t.Size()-1 { k, v := t.GetByIndex(idx - 1) - proof.Left = &PathWithNode{} - proof.Left.Path, _, _ = t.root.pathToKey(t, k) - proof.Left.Node = IAVLProofLeafNode{k, v} + path, node, _ := t.root.pathToKey(t, k) + proof.Left = &pathWithNode{ + Path: path, + Node: proofLeafNode{k, v, node.version}, + } } } diff --git a/iavl_proof_test.go b/proof_test.go similarity index 89% rename from iavl_proof_test.go rename to proof_test.go index 2fc5c7710..f5d2ee71e 100644 --- a/iavl_proof_test.go +++ b/proof_test.go @@ -11,19 +11,18 @@ import ( "testing" ) -func TestIAVLTreeGetWithProof(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) +func TestTreeGetWithProof(t *testing.T) { + tree := NewTree(0, nil) require := require.New(t) - keys := [][]byte{} for _, ikey := range []byte{0x11, 0x32, 0x50, 0x72, 0x99} { key := []byte{ikey} - keys = append(keys, key) tree.Set(key, []byte(randstr(8))) } root := tree.Hash() key := []byte{0x32} val, proof, err := tree.GetWithProof(key) + require.NoError(err) _, ok := proof.(*KeyExistsProof) require.True(ok) require.NotEmpty(val) @@ -34,6 +33,7 @@ func TestIAVLTreeGetWithProof(t *testing.T) { key = []byte{0x1} val, proof, err = tree.GetWithProof(key) + require.NoError(err) _, ok = proof.(*KeyAbsentProof) require.True(ok) require.Empty(val) @@ -43,16 +43,8 @@ func TestIAVLTreeGetWithProof(t *testing.T) { require.NoError(err) } -func reverseBytes(xs [][]byte) [][]byte { - reversed := [][]byte{} - for i := len(xs) - 1; i >= 0; i-- { - reversed = append(reversed, xs[i]) - } - return reversed -} - -func TestIAVLTreeKeyExistsProof(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) +func TestTreeKeyExistsProof(t *testing.T) { + tree := NewTree(0, nil) // should get false for proof with nil root _, proof, _ := tree.getWithProof([]byte("foo")) @@ -87,8 +79,8 @@ func TestIAVLTreeKeyExistsProof(t *testing.T) { // TODO: Test with single value in tree. } -func TestIAVLTreeKeyInRangeProofs(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) +func TestTreeKeyInRangeProofs(t *testing.T) { + tree := NewTree(0, nil) require := require.New(t) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, @@ -140,8 +132,8 @@ func TestIAVLTreeKeyInRangeProofs(t *testing.T) { } } -func TestIAVLTreeKeyFirstInRangeProofsVerify(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) +func TestTreeKeyFirstInRangeProofsVerify(t *testing.T) { + tree := NewTree(0, nil) require := require.New(t) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, @@ -169,12 +161,12 @@ func TestIAVLTreeKeyFirstInRangeProofsVerify(t *testing.T) { RootHash: root, PathToKey: dummyPathToKey(tree, []byte{0x72}), }, - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x50}), dummyLeafNode([]byte{0x11}, []byte{0x11}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 1: { root: root, @@ -183,12 +175,12 @@ func TestIAVLTreeKeyFirstInRangeProofsVerify(t *testing.T) { resultKey: []byte{0x21}, resultVal: []byte{0x21}, proof: &KeyFirstInRangeProof{ - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 2: { // Result is outside of the range (right). root: root, @@ -229,12 +221,12 @@ func TestIAVLTreeKeyFirstInRangeProofsVerify(t *testing.T) { RootHash: root, PathToKey: dummyPathToKey(tree, []byte{0x11}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xf7}), Node: dummyLeafNode([]byte{0xf7}, []byte{0xf7}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 5: { root: root, @@ -246,16 +238,16 @@ func TestIAVLTreeKeyFirstInRangeProofsVerify(t *testing.T) { KeyExistsProof: KeyExistsProof{ RootHash: root, }, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xa}), Node: dummyLeafNode([]byte{0xa}, []byte{0xa}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xf7}), Node: dummyLeafNode([]byte{0xf7}, []byte{0xf7}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 6: { root: root, @@ -268,16 +260,16 @@ func TestIAVLTreeKeyFirstInRangeProofsVerify(t *testing.T) { RootHash: root, PathToKey: dummyPathToKey(tree, []byte{0xa1}), }, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xa}), Node: dummyLeafNode([]byte{0xa}, []byte{0xa}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xe4}), Node: dummyLeafNode([]byte{0xe4}, []byte{0xe4}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 7: { root: root, @@ -286,12 +278,12 @@ func TestIAVLTreeKeyFirstInRangeProofsVerify(t *testing.T) { resultKey: []byte{0x29}, resultVal: []byte{0x29}, proof: &KeyFirstInRangeProof{ - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, } @@ -302,8 +294,8 @@ func TestIAVLTreeKeyFirstInRangeProofsVerify(t *testing.T) { } } -func TestIAVLTreeKeyLastInRangeProofsVerify(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) +func TestTreeKeyLastInRangeProofsVerify(t *testing.T) { + tree := NewTree(0, nil) require := require.New(t) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, @@ -332,7 +324,7 @@ func TestIAVLTreeKeyLastInRangeProofsVerify(t *testing.T) { PathToKey: dummyPathToKey(tree, []byte{0x11}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 1: { // Result is outside of the range (right). root: root, @@ -373,12 +365,12 @@ func TestIAVLTreeKeyLastInRangeProofsVerify(t *testing.T) { RootHash: root, PathToKey: dummyPathToKey(tree, []byte{0x11}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xf7}), Node: dummyLeafNode([]byte{0xf7}, []byte{0xf7}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 4: { root: root, @@ -390,16 +382,16 @@ func TestIAVLTreeKeyLastInRangeProofsVerify(t *testing.T) { KeyExistsProof: KeyExistsProof{ RootHash: root, }, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xa}), Node: dummyLeafNode([]byte{0xa}, []byte{0xa}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xf7}), Node: dummyLeafNode([]byte{0xf7}, []byte{0xf7}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 5: { root: root, @@ -412,16 +404,16 @@ func TestIAVLTreeKeyLastInRangeProofsVerify(t *testing.T) { RootHash: root, PathToKey: dummyPathToKey(tree, []byte{0xa1}), }, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xa}), Node: dummyLeafNode([]byte{0xa}, []byte{0xa}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xe4}), Node: dummyLeafNode([]byte{0xe4}, []byte{0xe4}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 6: { root: root, @@ -430,12 +422,12 @@ func TestIAVLTreeKeyLastInRangeProofsVerify(t *testing.T) { resultKey: []byte{0x29}, resultVal: []byte{0x29}, invalidProof: &KeyLastInRangeProof{ - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, } @@ -446,8 +438,8 @@ func TestIAVLTreeKeyLastInRangeProofsVerify(t *testing.T) { } } -func TestIAVLTreeKeyRangeProof(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) +func TestTreeKeyRangeProof(t *testing.T) { + tree := NewTree(0, nil) require := require.New(t) keys := [][]byte{} for _, ikey := range []byte{ @@ -532,18 +524,15 @@ func TestIAVLTreeKeyRangeProof(t *testing.T) { } } -func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) +func TestTreeKeyRangeProofVerify(t *testing.T) { + tree := NewTree(0, nil) require := require.New(t) assert := assert.New(t) - keys := [][]byte{} - values := [][]byte{} for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, } { key, val := []byte{ikey}, []byte{ikey} - keys, values = append(keys, key), append(values, val) tree.Set(key, val) } root := tree.Hash() @@ -561,7 +550,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { keyEnd: []byte{0xff}, root: root, invalidProof: &KeyRangeProof{RootHash: root}, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 1: { keyStart: []byte{0x0}, @@ -578,16 +567,16 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x99}), Node: dummyLeafNode([]byte{0x99}, []byte{0x99}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xa1}), Node: dummyLeafNode([]byte{0xa1}, []byte{0xa1}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 3: { // An invalid proof with one path. keyStart: []byte{0xf8}, @@ -595,12 +584,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xe4}), Node: dummyLeafNode([]byte{0xe4}, []byte{0xe4}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 4: { // An invalid proof with one path. keyStart: []byte{0x30}, @@ -608,12 +597,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xa}), Node: dummyLeafNode([]byte{0xa}, []byte{0xa}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 5: { // An invalid proof with one path. keyStart: []byte{0x1}, @@ -621,12 +610,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 6: { // An invalid proof with one path. keyStart: []byte{0x30}, @@ -634,12 +623,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x99}), Node: dummyLeafNode([]byte{0x99}, []byte{0x99}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 7: { keyStart: []byte{0x30}, @@ -647,16 +636,16 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xe4}), Node: dummyLeafNode([]byte{0xe4}, []byte{0xe4}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 8: { keyStart: []byte{0x30}, @@ -664,16 +653,16 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x01}, []byte{0x01}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x2e}), Node: dummyLeafNode([]byte{0x2e}, []byte{0x2e}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 9: { keyStart: []byte{0x30}, @@ -681,16 +670,16 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x2e}), Node: dummyLeafNode([]byte{0x2f}, []byte{0x2f}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 10: { keyStart: []byte{0x12}, @@ -701,12 +690,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { invalidProof: &KeyRangeProof{ RootHash: root, PathToKeys: []*PathToKey{dummyPathToKey(tree, []byte{0x2e})}, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 11: { keyStart: []byte{0x12}, @@ -717,12 +706,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { invalidProof: &KeyRangeProof{ RootHash: root, PathToKeys: []*PathToKey{dummyPathToKey(tree, []byte{0x2e})}, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 12: { keyStart: []byte{0x10}, @@ -733,12 +722,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { invalidProof: &KeyRangeProof{ RootHash: root, PathToKeys: []*PathToKey{dummyPathToKey(tree, []byte{0x2e})}, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x32}), Node: dummyLeafNode([]byte{0x32}, []byte{0x32}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 13: { // Construct an invalid proof with missing 0x2e and 0x32 keys. keyStart: []byte{0x11}, @@ -768,7 +757,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { dummyPathToKey(tree, []byte{0x2e}), dummyPathToKey(tree, []byte{0x32}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xf7}), Node: dummyLeafNode([]byte{0xf7}, []byte{0xf7}), }, @@ -788,7 +777,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { dummyPathToKey(tree, []byte{0x32}), dummyPathToKey(tree, []byte{0x50}), }, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0xa}), Node: dummyLeafNode([]byte{0xa}, []byte{0xa}), }, @@ -807,7 +796,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { dummyPathToKey(tree, []byte{0x11}).dropRoot(), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 17: { // An invalid proof with one path and a limit. keyStart: []byte{0x30}, @@ -816,12 +805,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { limit: 10, invalidProof: &KeyRangeProof{ RootHash: root, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x0a}), Node: dummyLeafNode([]byte{0x0a}, []byte{0x0a}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 18: { // An invalid proof with one path and a limit. keyStart: []byte{0x30}, @@ -830,12 +819,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { limit: 10, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x99}), Node: dummyLeafNode([]byte{0x99}, []byte{0x99}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 19: { // First value returned is wrong. Should be 0x11. keyStart: []byte{0x10}, @@ -850,7 +839,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { dummyPathToKey(tree, []byte{0x2e}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 20: { // Ethan Frey's failing test case. keyStart: []byte{0x05}, @@ -861,7 +850,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, @@ -869,12 +858,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { dummyPathToKey(tree, []byte{0x2e}), dummyPathToKey(tree, []byte{0x32}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x50}), Node: dummyLeafNode([]byte{0x50}, []byte{0x50}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 21: { // Ethan Frey's reverse failing test case. keyStart: []byte{0xca}, @@ -885,7 +874,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, @@ -895,7 +884,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { }, Right: nil, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 22: { // Partial results are detected if they don't fill the limit. keyStart: []byte{0x05}, @@ -906,7 +895,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { root: root, invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x11}), Node: dummyLeafNode([]byte{0x11}, []byte{0x11}), }, @@ -914,12 +903,12 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { dummyPathToKey(tree, []byte{0x2e}), dummyPathToKey(tree, []byte{0x32}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x50}), Node: dummyLeafNode([]byte{0x50}, []byte{0x50}), }, }, - expectedError: errInvalidProof, + expectedError: ErrInvalidProof, }, 23: { // Valid proof. keyStart: []byte{0x10}, @@ -931,7 +920,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x0a}), Node: dummyLeafNode([]byte{0x0a}, []byte{0x0a}), }, @@ -940,7 +929,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { dummyPathToKey(tree, []byte{0x2e}), dummyPathToKey(tree, []byte{0x32}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x50}), Node: dummyLeafNode([]byte{0x50}, []byte{0x50}), }, @@ -957,7 +946,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { invalidProof: &KeyRangeProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x0a}), Node: dummyLeafNode([]byte{0x0a}, []byte{0x0a}), }, @@ -966,7 +955,7 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { dummyPathToKey(tree, []byte{0x2e}), dummyPathToKey(tree, []byte{0x32}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ Path: dummyPathToKey(tree, []byte{0x50}), Node: dummyLeafNode([]byte{0x50}, []byte{0x50}), }, @@ -1007,8 +996,8 @@ func TestIAVLTreeKeyRangeProofVerify(t *testing.T) { } } -func TestIAVLTreeKeyAbsentProof(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) +func TestTreeKeyAbsentProof(t *testing.T) { + tree := NewTree(0, nil) require := require.New(t) proof, err := tree.keyAbsentProof([]byte{0x1}) @@ -1035,7 +1024,7 @@ func TestIAVLTreeKeyAbsentProof(t *testing.T) { exists := false for _, k := range keys { - if bytes.Compare(key, k) == 0 { + if bytes.Equal(key, k) { exists = true break } @@ -1068,7 +1057,7 @@ func TestIAVLTreeKeyAbsentProof(t *testing.T) { } func TestKeyAbsentProofVerify(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) + tree := NewTree(0, nil) require := require.New(t) allKeys := []byte{0x11, 0x32, 0x50, 0x72, 0x99} for _, ikey := range allKeys { @@ -1089,11 +1078,11 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: []byte{0x32, 0x50, 0x99, 0x0, 0xff}, proof: &KeyAbsentProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x32}), dummyLeafNode([]byte{0x32}, []byte{0x32}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ dummyPathToKey(tree, []byte{0x50}), dummyLeafNode([]byte{0x50}, []byte{0x50}), }, @@ -1105,7 +1094,7 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: []byte{0x99, 0x91, 0x0}, proof: &KeyAbsentProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x99}), dummyLeafNode([]byte{0x99}, []byte{0x99}), }, @@ -1117,7 +1106,7 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: []byte{0x11, 0x99, 0x12}, proof: &KeyAbsentProof{ RootHash: root, - Right: &PathWithNode{ + Right: &pathWithNode{ dummyPathToKey(tree, []byte{0x11}), dummyLeafNode([]byte{0x11}, []byte{0x11}), }, @@ -1129,7 +1118,7 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: allKeys, proof: &KeyAbsentProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x32}), dummyLeafNode([]byte{0x32}, []byte{0x32}), }, @@ -1141,7 +1130,7 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: allKeys, proof: &KeyAbsentProof{ RootHash: root, - Right: &PathWithNode{ + Right: &pathWithNode{ dummyPathToKey(tree, []byte{0x32}), dummyLeafNode([]byte{0x32}, []byte{0x32}), }, @@ -1153,11 +1142,11 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: allKeys, proof: &KeyAbsentProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x11}), dummyLeafNode([]byte{0x11}, []byte{0x11}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ dummyPathToKey(tree, []byte{0x50}), dummyLeafNode([]byte{0x50}, []byte{0x50}), }, @@ -1169,11 +1158,11 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: allKeys, proof: &KeyAbsentProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x50}), dummyLeafNode([]byte{0x50}, []byte{0x50}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ dummyPathToKey(tree, []byte{0x32}), dummyLeafNode([]byte{0x32}, []byte{0x32}), }, @@ -1185,11 +1174,11 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: allKeys, proof: &KeyAbsentProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x32}), dummyLeafNode([]byte{0x32}, []byte{0x32}), }, - Right: &PathWithNode{ + Right: &pathWithNode{ dummyPathToKey(tree, []byte{0x32}), dummyLeafNode([]byte{0x32}, []byte{0x32}), }, @@ -1209,7 +1198,7 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: allKeys, proof: &KeyAbsentProof{ RootHash: []byte(randstr(32)), - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x99}), dummyLeafNode([]byte{0x99}, []byte{0x99}), }, @@ -1221,7 +1210,7 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: allKeys, proof: &KeyAbsentProof{ RootHash: root, - Left: &PathWithNode{ + Left: &pathWithNode{ dummyPathToKey(tree, []byte{0x99}), dummyLeafNode([]byte{0x90}, []byte{0x90}), }, @@ -1233,7 +1222,7 @@ func TestKeyAbsentProofVerify(t *testing.T) { invalidKeys: allKeys, proof: &KeyAbsentProof{ RootHash: root, - Right: &PathWithNode{ + Right: &pathWithNode{ dummyPathToKey(tree, []byte{0x11}), dummyLeafNode([]byte{0x12}, []byte{0x12}), }, diff --git a/testutils_test.go b/testutils_test.go new file mode 100644 index 000000000..6f44a430d --- /dev/null +++ b/testutils_test.go @@ -0,0 +1,185 @@ +package iavl + +import ( + "fmt" + mrand "math/rand" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/go-wire" + . "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tmlibs/db" + . "github.com/tendermint/tmlibs/test" + + "runtime" + "testing" +) + +func dummyPathToKey(t *Tree, key []byte) *PathToKey { + path, _, err := t.root.pathToKey(t, key) + if err != nil { + panic(err) + } + return path +} + +func dummyLeafNode(key, val []byte) proofLeafNode { + return proofLeafNode{key, val, 0} +} + +func randstr(length int) string { + return RandStr(length) +} + +func i2b(i int) []byte { + bz := make([]byte, 4) + wire.PutInt32(bz, int32(i)) + return bz +} + +func b2i(bz []byte) int { + i := wire.GetInt32(bz) + return int(i) +} + +// Convenience for a new node +func N(l, r interface{}) *Node { + var left, right *Node + if _, ok := l.(*Node); ok { + left = l.(*Node) + } else { + left = NewNode(i2b(l.(int)), nil) + } + if _, ok := r.(*Node); ok { + right = r.(*Node) + } else { + right = NewNode(i2b(r.(int)), nil) + } + + n := &Node{ + key: right.lmd(nil).key, + value: nil, + leftNode: left, + rightNode: right, + } + n.calcHeightAndSize(nil) + return n +} + +// Setup a deep node +func T(n *Node) *Tree { + d := db.NewDB("test", db.MemDBBackendStr, "") + t := NewTree(0, d) + + n.hashWithCount() + t.root = n + return t +} + +// Convenience for simple printing of keys & tree structure +func P(n *Node) string { + if n.height == 0 { + return fmt.Sprintf("%v", b2i(n.key)) + } else { + return fmt.Sprintf("(%v %v)", P(n.leftNode), P(n.rightNode)) + } +} + +func randBytes(length int) []byte { + key := make([]byte, length) + // math.rand.Read always returns err=nil + mrand.Read(key) + return key +} + +type traverser struct { + first string + last string + count int +} + +func (t *traverser) view(key, value []byte) bool { + if t.first == "" { + t.first = string(key) + } + t.last = string(key) + t.count += 1 + return false +} + +func expectTraverse(t *testing.T, trav traverser, start, end string, count int) { + if trav.first != start { + t.Error("Bad start", start, trav.first) + } + if trav.last != end { + t.Error("Bad end", end, trav.last) + } + if trav.count != count { + t.Error("Bad count", count, trav.count) + } +} + +func testProof(t *testing.T, proof *KeyExistsProof, keyBytes, valueBytes, rootHashBytes []byte) { + // Proof must verify. + require.NoError(t, proof.Verify(keyBytes, valueBytes, rootHashBytes)) + + // Write/Read then verify. + proofBytes := proof.Bytes() + proof2, err := ReadKeyExistsProof(proofBytes) + require.Nil(t, err, "Failed to read KeyExistsProof from bytes: %v", err) + require.NoError(t, proof2.Verify(keyBytes, valueBytes, proof.RootHash)) + + // Random mutations must not verify + for i := 0; i < 10; i++ { + badProofBytes := MutateByteSlice(proofBytes) + badProof, err := ReadKeyExistsProof(badProofBytes) + // may be invalid... errors are okay + if err == nil { + assert.Error(t, badProof.Verify(keyBytes, valueBytes, rootHashBytes), + "Proof was still valid after a random mutation:\n%X\n%X", + proofBytes, badProofBytes) + } + } + + // targetted changes fails... + proof.RootHash = MutateByteSlice(proof.RootHash) + assert.Error(t, proof.Verify(keyBytes, valueBytes, rootHashBytes)) +} + +func BenchmarkImmutableAvlTreeMemDB(b *testing.B) { + db := db.NewDB("test", db.MemDBBackendStr, "") + benchmarkImmutableAvlTreeWithDB(b, db) +} + +func benchmarkImmutableAvlTreeWithDB(b *testing.B, db db.DB) { + defer db.Close() + + b.StopTimer() + + v := uint64(1) + t := NewVersionedTree(100000, db) + for i := 0; i < 1000000; i++ { + t.Set(i2b(int(RandInt32())), nil) + if i > 990000 && i%1000 == 999 { + t.SaveVersion(v) + v++ + } + } + b.ReportAllocs() + t.SaveVersion(v) + + fmt.Println("ok, starting") + + runtime.GC() + + b.StartTimer() + for i := 0; i < b.N; i++ { + ri := i2b(int(RandInt32())) + t.Set(ri, nil) + t.Remove(ri) + if i%100 == 99 { + t.SaveVersion(v) + v++ + } + } +} diff --git a/tree.go b/tree.go new file mode 100644 index 000000000..cd4d5f04d --- /dev/null +++ b/tree.go @@ -0,0 +1,247 @@ +package iavl + +import ( + "fmt" + "strings" + + cmn "github.com/tendermint/tmlibs/common" + dbm "github.com/tendermint/tmlibs/db" + + "github.com/pkg/errors" +) + +// Tree is an immutable AVL+ Tree. Note that this tree is not thread-safe. +type Tree struct { + root *Node + ndb *nodeDB +} + +// NewTree creates both im-memory and persistent instances +func NewTree(cacheSize int, db dbm.DB) *Tree { + if db == nil { + // In-memory Tree. + return &Tree{} + } + return &Tree{ + // NodeDB-backed Tree. + ndb: newNodeDB(cacheSize, db), + } +} + +// String returns a string representation of Tree. +func (t *Tree) String() string { + leaves := []string{} + t.Iterate(func(key []byte, val []byte) (stop bool) { + leaves = append(leaves, fmt.Sprintf("%x: %x", key, val)) + return false + }) + return "Tree{" + strings.Join(leaves, ", ") + "}" +} + +// Size returns the number of leaf nodes in the tree. +func (t *Tree) Size() int { + if t.root == nil { + return 0 + } + return t.root.size +} + +// Height returns the height of the tree. +func (t *Tree) Height() int8 { + if t.root == nil { + return 0 + } + return t.root.height +} + +// Has returns whether or not a key exists. +func (t *Tree) Has(key []byte) bool { + if t.root == nil { + return false + } + return t.root.has(t, key) +} + +// Set a key. Nil values are not supported. +func (t *Tree) Set(key []byte, value []byte) (updated bool) { + _, updated = t.set(key, value) + return updated +} + +func (t *Tree) set(key []byte, value []byte) (orphaned []*Node, updated bool) { + if value == nil { + cmn.PanicSanity(cmn.Fmt("Attempt to store nil value at key '%s'", key)) + } + if t.root == nil { + t.root = NewNode(key, value) + return nil, false + } + t.root, updated, orphaned = t.root.set(t, key, value) + + return orphaned, updated +} + +// Hash returns the root hash. +func (t *Tree) Hash() []byte { + if t.root == nil { + return nil + } + hash, _ := t.root.hashWithCount() + return hash +} + +// hashWithCount returns the root hash and hash count. +func (t *Tree) hashWithCount() ([]byte, int) { + if t.root == nil { + return nil, 0 + } + return t.root.hashWithCount() +} + +// Get returns the index and value of the specified key if it exists, or nil +// and the next index, if it doesn't. +func (t *Tree) Get(key []byte) (index int, value []byte) { + if t.root == nil { + return 0, nil + } + return t.root.get(t, key) +} + +// GetByIndex gets the key and value at the specified index. +func (t *Tree) GetByIndex(index int) (key []byte, value []byte) { + if t.root == nil { + return nil, nil + } + return t.root.getByIndex(t, index) +} + +// GetWithProof gets the value under the key if it exists, or returns nil. +// A proof of existence or absence is returned alongside the value. +func (t *Tree) GetWithProof(key []byte) ([]byte, KeyProof, error) { + value, eproof, err := t.getWithProof(key) + if err == nil { + return value, eproof, nil + } + + aproof, err := t.keyAbsentProof(key) + if err == nil { + return nil, aproof, nil + } + return nil, nil, errors.Wrap(err, "could not construct any proof") +} + +// GetRangeWithProof gets key/value pairs within the specified range and limit. To specify a descending +// range, swap the start and end keys. +// +// Returns a list of keys, a list of values and a proof. +func (t *Tree) GetRangeWithProof(startKey []byte, endKey []byte, limit int) ([][]byte, [][]byte, *KeyRangeProof, error) { + return t.getRangeWithProof(startKey, endKey, limit) +} + +// GetFirstInRangeWithProof gets the first key/value pair in the specified range, with a proof. +func (t *Tree) GetFirstInRangeWithProof(startKey, endKey []byte) ([]byte, []byte, *KeyFirstInRangeProof, error) { + return t.getFirstInRangeWithProof(startKey, endKey) +} + +// GetLastInRangeWithProof gets the last key/value pair in the specified range, with a proof. +func (t *Tree) GetLastInRangeWithProof(startKey, endKey []byte) ([]byte, []byte, *KeyLastInRangeProof, error) { + return t.getLastInRangeWithProof(startKey, endKey) +} + +// Remove tries to remove a key from the tree and if removed, returns its +// value, and 'true'. +func (t *Tree) Remove(key []byte) ([]byte, bool) { + value, _, removed := t.remove(key) + return value, removed +} + +// remove tries to remove a key from the tree and if removed, returns its +// value, nodes orphaned and 'true'. +func (t *Tree) remove(key []byte) (value []byte, orphans []*Node, removed bool) { + if t.root == nil { + return nil, nil, false + } + newRootHash, newRoot, _, value, orphaned := t.root.remove(t, key) + if len(orphaned) == 0 { + return nil, nil, false + } + + if newRoot == nil && newRootHash != nil { + t.root = t.ndb.GetNode(newRootHash) + } else { + t.root = newRoot + } + return value, orphaned, true +} + +// Iterate iterates over all keys of the tree, in order. +func (t *Tree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { + if t.root == nil { + return false + } + return t.root.traverse(t, true, func(node *Node) bool { + if node.height == 0 { + return fn(node.key, node.value) + } else { + return false + } + }) +} + +// IterateRange makes a callback for all nodes with key between start and end non-inclusive. +// If either are nil, then it is open on that side (nil, nil is the same as Iterate) +func (t *Tree) IterateRange(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { + if t.root == nil { + return false + } + return t.root.traverseInRange(t, start, end, ascending, false, func(node *Node) bool { + if node.height == 0 { + return fn(node.key, node.value) + } else { + return false + } + }) +} + +// IterateRangeInclusive makes a callback for all nodes with key between start and end inclusive. +// If either are nil, then it is open on that side (nil, nil is the same as Iterate) +func (t *Tree) IterateRangeInclusive(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { + if t.root == nil { + return false + } + return t.root.traverseInRange(t, start, end, ascending, true, func(node *Node) bool { + if node.height == 0 { + return fn(node.key, node.value) + } else { + return false + } + }) +} + +// Clone creates a clone of the tree. Used internally by VersionedTree. +func (tree *Tree) clone() *Tree { + return &Tree{ + root: tree.root, + ndb: tree.ndb, + } +} + +// Load the tree from disk, from the given root hash, including all orphans. +// Used internally by VersionedTree. +func (tree *Tree) load(root []byte) { + if len(root) == 0 { + tree.root = nil + return + } + tree.root = tree.ndb.GetNode(root) +} + +// nodeSize is like Size, but includes inner nodes too. +func (t *Tree) nodeSize() int { + size := 0 + t.root.traverse(t, true, func(n *Node) bool { + size++ + return false + }) + return size +} diff --git a/iavl_tree_dotgraph.go b/tree_dotgraph.go similarity index 88% rename from iavl_tree_dotgraph.go rename to tree_dotgraph.go index c4114edc2..57fc21678 100644 --- a/iavl_tree_dotgraph.go +++ b/tree_dotgraph.go @@ -41,11 +41,11 @@ var defaultGraphNodeAttrs = map[string]string{ "shape": "circle", } -func WriteDOTGraph(w io.Writer, tree *IAVLTree, paths []*PathToKey) { +func WriteDOTGraph(w io.Writer, tree *Tree, paths []*PathToKey) { ctx := &graphContext{} tree.root.hashWithCount() - tree.root.traverse(tree, true, func(node *IAVLNode) bool { + tree.root.traverse(tree, true, func(node *Node) bool { graphNode := &graphNode{ Attrs: map[string]string{}, Hash: fmt.Sprintf("%x", node.hash), @@ -55,8 +55,9 @@ func WriteDOTGraph(w io.Writer, tree *IAVLTree, paths []*PathToKey) { } shortHash := graphNode.Hash[:7] - graphNode.Label = mkLabel(fmt.Sprintf("%x", node.key), 16, "sans-serif") + graphNode.Label = mkLabel(fmt.Sprintf("%s", node.key), 16, "sans-serif") graphNode.Label += mkLabel(shortHash, 10, "monospace") + graphNode.Label += mkLabel(fmt.Sprintf("version=%d", node.version), 10, "monospace") if node.value != nil { graphNode.Label += mkLabel(string(node.value), 10, "sans-serif") diff --git a/iavl_tree_dotgraph_test.go b/tree_dotgraph_test.go similarity index 87% rename from iavl_tree_dotgraph_test.go rename to tree_dotgraph_test.go index e963b3487..216ff483c 100644 --- a/iavl_tree_dotgraph_test.go +++ b/tree_dotgraph_test.go @@ -6,7 +6,7 @@ import ( ) func TestWriteDOTGraph(t *testing.T) { - var tree *IAVLTree = NewIAVLTree(0, nil) + var tree *Tree = NewTree(0, nil) for _, ikey := range []byte{ 0x0a, 0x11, 0x2e, 0x32, 0x50, 0x72, 0x99, 0xa1, 0xe4, 0xf7, } { diff --git a/tree_fuzz_test.go b/tree_fuzz_test.go new file mode 100644 index 000000000..e13ccebdb --- /dev/null +++ b/tree_fuzz_test.go @@ -0,0 +1,124 @@ +package iavl + +import ( + "fmt" + "testing" + + "github.com/pkg/errors" + "github.com/tendermint/tmlibs/db" + + cmn "github.com/tendermint/tmlibs/common" +) + +// This file implement fuzz testing by generating programs and then running +// them. If an error occurs, the program that had the error is printed. + +// A program is a list of instructions. +type program struct { + instructions []instruction +} + +func (p *program) Execute(tree *VersionedTree) (err error) { + var errLine int + + defer func() { + if r := recover(); r != nil { + var str string + + for i, instr := range p.instructions { + prefix := " " + if i == errLine { + prefix = ">> " + } + str += prefix + instr.String() + "\n" + } + err = errors.Errorf("Program panicked with: %s\n%s", r, str) + } + }() + + for i, instr := range p.instructions { + errLine = i + instr.Execute(tree) + } + return +} + +func (p *program) addInstruction(i instruction) { + p.instructions = append(p.instructions, i) +} + +func (prog *program) size() int { + return len(prog.instructions) +} + +type instruction struct { + op string + k, v []byte + version uint64 +} + +func (i instruction) Execute(tree *VersionedTree) { + switch i.op { + case "SET": + tree.Set(i.k, i.v) + case "REMOVE": + tree.Remove(i.k) + case "SAVE": + tree.SaveVersion(i.version) + case "DELETE": + tree.DeleteVersion(i.version) + default: + panic("Unrecognized op: " + i.op) + } +} + +func (i instruction) String() string { + if i.version > 0 { + return fmt.Sprintf("%-8s %-8s %-8s %-8d", i.op, i.k, i.v, i.version) + } + return fmt.Sprintf("%-8s %-8s %-8s", i.op, i.k, i.v) +} + +// Generate a random program of the given size. +func genRandomProgram(size int) *program { + p := &program{} + nextVersion := 1 + + for p.size() < size { + k, v := []byte(cmn.RandStr(1)), []byte(cmn.RandStr(1)) + + switch cmn.RandInt() % 7 { + case 0, 1, 2: + p.addInstruction(instruction{op: "SET", k: k, v: v}) + case 3, 4: + p.addInstruction(instruction{op: "REMOVE", k: k}) + case 5: + p.addInstruction(instruction{op: "SAVE", version: uint64(nextVersion)}) + nextVersion++ + case 6: + if rv := cmn.RandInt() % nextVersion; rv < nextVersion && rv > 0 { + p.addInstruction(instruction{op: "DELETE", version: uint64(rv)}) + } + } + } + return p +} + +// Generate many programs and run them. +func TestVersionedTreeFuzz(t *testing.T) { + maxIterations := testFuzzIterations + progsPerIteration := 100000 + iterations := 0 + + for size := 5; iterations < maxIterations; size++ { + for i := 0; i < progsPerIteration/size; i++ { + tree := NewVersionedTree(0, db.NewMemDB()) + program := genRandomProgram(size) + err := program.Execute(tree) + if err != nil { + t.Fatalf("Error after %d iterations (size %d): %s\n%s", iterations, size, err.Error(), tree.String()) + } + iterations++ + } + } +} diff --git a/tree_test.go b/tree_test.go new file mode 100644 index 000000000..ae06f7f7d --- /dev/null +++ b/tree_test.go @@ -0,0 +1,1049 @@ +package iavl + +import ( + "bytes" + "flag" + "math/rand" + "os" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tendermint/tmlibs/db" + + cmn "github.com/tendermint/tmlibs/common" +) + +var testLevelDB bool +var testFuzzIterations int + +func init() { + flag.BoolVar(&testLevelDB, "test.leveldb", false, "test leveldb backend") + flag.IntVar(&testFuzzIterations, "test.fuzz-iterations", 100000, "number of fuzz testing iterations") + flag.Parse() +} + +func getTestDB() (db.DB, func()) { + if testLevelDB { + d, err := db.NewGoLevelDB("test", ".") + if err != nil { + panic(err) + } + return d, func() { + d.Close() + os.RemoveAll("./test.db") + } + } + return db.NewMemDB(), func() {} +} + +func TestVersionedRandomTree(t *testing.T) { + require := require.New(t) + + d, closeDB := getTestDB() + defer closeDB() + + tree := NewVersionedTree(100, d) + versions := 50 + keysPerVersion := 30 + + // Create a tree of size 1000 with 100 versions. + for i := 1; i <= versions; i++ { + for j := 0; j < keysPerVersion; j++ { + k := []byte(cmn.RandStr(8)) + v := []byte(cmn.RandStr(8)) + tree.Set(k, v) + } + tree.SaveVersion(uint64(i)) + } + require.Equal(versions, len(tree.ndb.roots()), "wrong number of roots") + require.Equal(versions*keysPerVersion, len(tree.ndb.leafNodes()), "wrong number of nodes") + + // Before deleting old versions, we should have equal or more nodes in the + // db than in the current tree version. + require.True(len(tree.ndb.nodes()) >= tree.nodeSize()) + + for i := 1; i < versions; i++ { + tree.DeleteVersion(uint64(i)) + } + + require.Len(tree.versions, 1, "tree must have one version left") + require.Equal(tree.versions[uint64(versions)].root, tree.root) + + // After cleaning up all previous versions, we should have as many nodes + // in the db as in the current tree version. + require.Len(tree.ndb.leafNodes(), tree.Size()) + + require.Equal(tree.nodeSize(), len(tree.ndb.nodes())) +} + +func TestVersionedRandomTreeSmallKeys(t *testing.T) { + require := require.New(t) + d, closeDB := getTestDB() + defer closeDB() + + tree := NewVersionedTree(100, d) + singleVersionTree := NewVersionedTree(0, db.NewMemDB()) + versions := 20 + keysPerVersion := 50 + + for i := 1; i <= versions; i++ { + for j := 0; j < keysPerVersion; j++ { + // Keys of size one are likely to be overwritten. + k := []byte(cmn.RandStr(1)) + v := []byte(cmn.RandStr(8)) + tree.Set(k, v) + singleVersionTree.Set(k, v) + } + tree.SaveVersion(uint64(i)) + } + singleVersionTree.SaveVersion(1) + + for i := 1; i < versions; i++ { + tree.DeleteVersion(uint64(i)) + } + + // After cleaning up all previous versions, we should have as many nodes + // in the db as in the current tree version. The simple tree must be equal + // too. + require.Len(tree.ndb.leafNodes(), tree.Size()) + require.Len(tree.ndb.nodes(), tree.nodeSize()) + require.Len(tree.ndb.nodes(), singleVersionTree.nodeSize()) + + // Try getting random keys. + for i := 0; i < keysPerVersion; i++ { + _, val := tree.Get([]byte(cmn.RandStr(1))) + require.NotNil(val) + require.NotEmpty(val) + } +} + +func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { + require := require.New(t) + d, closeDB := getTestDB() + defer closeDB() + + tree := NewVersionedTree(100, d) + singleVersionTree := NewVersionedTree(0, db.NewMemDB()) + versions := 30 + keysPerVersion := 50 + + for i := 1; i <= versions; i++ { + for j := 0; j < keysPerVersion; j++ { + // Keys of size one are likely to be overwritten. + k := []byte(cmn.RandStr(1)) + v := []byte(cmn.RandStr(8)) + tree.Set(k, v) + singleVersionTree.Set(k, v) + } + tree.SaveVersion(uint64(i)) + } + singleVersionTree.SaveVersion(1) + + for _, i := range rand.Perm(versions - 1) { + tree.DeleteVersion(uint64(i + 1)) + } + + // After cleaning up all previous versions, we should have as many nodes + // in the db as in the current tree version. The simple tree must be equal + // too. + require.Len(tree.ndb.leafNodes(), tree.Size()) + require.Len(tree.ndb.nodes(), tree.nodeSize()) + require.Len(tree.ndb.nodes(), singleVersionTree.nodeSize()) + + // Try getting random keys. + for i := 0; i < keysPerVersion; i++ { + _, val := tree.Get([]byte(cmn.RandStr(1))) + require.NotNil(val) + require.NotEmpty(val) + } +} + +func TestVersionedTreeSpecial1(t *testing.T) { + tree := NewVersionedTree(100, db.NewMemDB()) + + tree.Set([]byte("C"), []byte("so43QQFN")) + tree.SaveVersion(1) + + tree.Set([]byte("A"), []byte("ut7sTTAO")) + tree.SaveVersion(2) + + tree.Set([]byte("X"), []byte("AoWWC1kN")) + tree.SaveVersion(3) + + tree.Set([]byte("T"), []byte("MhkWjkVy")) + tree.SaveVersion(4) + + tree.DeleteVersion(1) + tree.DeleteVersion(2) + tree.DeleteVersion(3) + + require.Equal(t, tree.nodeSize(), len(tree.ndb.nodes())) +} + +func TestVersionedRandomTreeSpecial2(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(100, db.NewMemDB()) + + tree.Set([]byte("OFMe2Yvm"), []byte("ez2OtQtE")) + tree.Set([]byte("WEN4iN7Y"), []byte("kQNyUalI")) + tree.SaveVersion(1) + + tree.Set([]byte("1yY3pXHr"), []byte("udYznpII")) + tree.Set([]byte("7OSHNE7k"), []byte("ff181M2d")) + tree.SaveVersion(2) + + tree.DeleteVersion(1) + require.Len(tree.ndb.nodes(), tree.nodeSize()) +} + +func TestVersionedTree(t *testing.T) { + require := require.New(t) + d, closeDB := getTestDB() + defer closeDB() + + tree := NewVersionedTree(100, d) + + // We start with zero keys in the databse. + require.Equal(0, tree.ndb.size()) + require.True(tree.IsEmpty()) + + // version 0 + + tree.Set([]byte("key1"), []byte("val0")) + tree.Set([]byte("key2"), []byte("val0")) + + // Still zero keys, since we haven't written them. + require.Len(tree.ndb.leafNodes(), 0) + require.False(tree.IsEmpty()) + + // Saving with version zero is an error. + _, err := tree.SaveVersion(0) + require.Error(err) + + // Now let's write the keys to storage. + hash1, err := tree.SaveVersion(1) + require.NoError(err) + require.False(tree.IsEmpty()) + + // Saving twice with the same version is an error. + _, err = tree.SaveVersion(1) + require.Error(err) + + // -----1----- + // key1 = val0 + // key2 = val0 + // ----------- + + nodes1 := tree.ndb.leafNodes() + require.Len(nodes1, 2, "db should have a size of 2") + + // version 1 + + tree.Set([]byte("key1"), []byte("val1")) + tree.Set([]byte("key2"), []byte("val1")) + tree.Set([]byte("key3"), []byte("val1")) + require.Len(tree.ndb.leafNodes(), len(nodes1)) + + hash2, err := tree.SaveVersion(2) + require.NoError(err) + require.False(bytes.Equal(hash1, hash2)) + + // Recreate a new tree and load it, to make sure it works in this + // scenario. + tree = NewVersionedTree(100, d) + require.NoError(tree.Load()) + + require.Len(tree.versions, 2, "wrong number of versions") + + // -----1----- + // key1 = val0 + // key2 = val0 + // -----2----- + // key1 = val1 + // key2 = val1 + // key3 = val1 + // ----------- + + nodes2 := tree.ndb.leafNodes() + require.Len(nodes2, 5, "db should have grown in size") + require.Len(tree.ndb.orphans(), 3, "db should have three orphans") + + // Create two more orphans. + tree.Remove([]byte("key1")) + tree.Set([]byte("key2"), []byte("val2")) + + hash3, _ := tree.SaveVersion(3) + + // -----1----- + // key1 = val0 (replaced) + // key2 = val0 (replaced) + // -----2----- + // key1 = val1 (removed) + // key2 = val1 (replaced) + // key3 = val1 + // -----3----- + // key2 = val2 + // ----------- + + nodes3 := tree.ndb.leafNodes() + require.Len(nodes3, 6, "wrong number of nodes") + require.Len(tree.ndb.orphans(), 6, "wrong number of orphans") + + hash4, _ := tree.SaveVersion(4) + require.EqualValues(hash3, hash4) + require.NotNil(hash4) + + tree = NewVersionedTree(100, d) + require.NoError(tree.Load()) + + // ------------ + // DB UNCHANGED + // ------------ + + nodes4 := tree.ndb.leafNodes() + require.Len(nodes4, len(nodes3), "db should not have changed in size") + + tree.Set([]byte("key1"), []byte("val0")) + + // "key2" + _, val := tree.GetVersioned([]byte("key2"), 0) + require.Nil(val) + + _, val = tree.GetVersioned([]byte("key2"), 1) + require.Equal("val0", string(val)) + + _, val = tree.GetVersioned([]byte("key2"), 2) + require.Equal("val1", string(val)) + + _, val = tree.Get([]byte("key2")) + require.Equal("val2", string(val)) + + // "key1" + _, val = tree.GetVersioned([]byte("key1"), 1) + require.Equal("val0", string(val)) + + _, val = tree.GetVersioned([]byte("key1"), 2) + require.Equal("val1", string(val)) + + _, val = tree.GetVersioned([]byte("key1"), 3) + require.Nil(val) + + _, val = tree.GetVersioned([]byte("key1"), 4) + require.Nil(val) + + _, val = tree.Get([]byte("key1")) + require.Equal("val0", string(val)) + + // "key3" + _, val = tree.GetVersioned([]byte("key3"), 0) + require.Nil(val) + + _, val = tree.GetVersioned([]byte("key3"), 2) + require.Equal("val1", string(val)) + + _, val = tree.GetVersioned([]byte("key3"), 3) + require.Equal("val1", string(val)) + + // Delete a version. After this the keys in that version should not be found. + + tree.DeleteVersion(2) + + // -----1----- + // key1 = val0 + // key2 = val0 + // -----2----- + // key3 = val1 + // -----3----- + // key2 = val2 + // ----------- + + nodes5 := tree.ndb.leafNodes() + require.True(len(nodes5) < len(nodes4), "db should have shrunk after delete") + + _, val = tree.GetVersioned([]byte("key2"), 2) + require.Nil(val) + + _, val = tree.GetVersioned([]byte("key3"), 2) + require.Nil(val) + + // But they should still exist in the latest version. + + _, val = tree.Get([]byte("key2")) + require.Equal("val2", string(val)) + + _, val = tree.Get([]byte("key3")) + require.Equal("val1", string(val)) + + // Version 1 should still be available. + + _, val = tree.GetVersioned([]byte("key1"), 1) + require.Equal("val0", string(val)) + + _, val = tree.GetVersioned([]byte("key2"), 1) + require.Equal("val0", string(val)) +} + +func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { + d, closeDB := getTestDB() + defer closeDB() + + tree := NewVersionedTree(0, d) + + tree.Set([]byte("key0"), []byte("val0")) + tree.Set([]byte("key1"), []byte("val0")) + tree.Set([]byte("key2"), []byte("val0")) + tree.SaveVersion(1) + + require.Len(t, tree.ndb.leafNodes(), 3) + + tree.Set([]byte("key1"), []byte("val1")) + tree.Set([]byte("key2"), []byte("val1")) + tree.Set([]byte("key3"), []byte("val1")) + tree.SaveVersion(2) + + require.Len(t, tree.ndb.leafNodes(), 6) + + tree.Set([]byte("key0"), []byte("val2")) + tree.Remove([]byte("key1")) + tree.Set([]byte("key2"), []byte("val2")) + tree.SaveVersion(3) + + require.Len(t, tree.ndb.leafNodes(), 8) + + tree.DeleteVersion(2) + + require.Len(t, tree.ndb.leafNodes(), 6) + + tree.DeleteVersion(1) + + require.Len(t, tree.ndb.leafNodes(), 3) + + tree2 := NewVersionedTree(0, db.NewMemDB()) + tree2.Set([]byte("key0"), []byte("val2")) + tree2.Set([]byte("key2"), []byte("val2")) + tree2.Set([]byte("key3"), []byte("val1")) + tree2.SaveVersion(1) + + require.Equal(t, tree2.nodeSize(), tree.nodeSize()) +} + +func TestVersionedTreeOrphanDeleting(t *testing.T) { + tree := NewVersionedTree(0, db.NewMemDB()) + + tree.Set([]byte("key0"), []byte("val0")) + tree.Set([]byte("key1"), []byte("val0")) + tree.Set([]byte("key2"), []byte("val0")) + tree.SaveVersion(1) + + tree.Set([]byte("key1"), []byte("val1")) + tree.Set([]byte("key2"), []byte("val1")) + tree.Set([]byte("key3"), []byte("val1")) + tree.SaveVersion(2) + + tree.Set([]byte("key0"), []byte("val2")) + tree.Remove([]byte("key1")) + tree.Set([]byte("key2"), []byte("val2")) + tree.SaveVersion(3) + tree.DeleteVersion(2) + + _, val := tree.Get([]byte("key0")) + require.Equal(t, val, []byte("val2")) + + _, val = tree.Get([]byte("key1")) + require.Nil(t, val) + + _, val = tree.Get([]byte("key2")) + require.Equal(t, val, []byte("val2")) + + _, val = tree.Get([]byte("key3")) + require.Equal(t, val, []byte("val1")) + + tree.DeleteVersion(1) + + require.Len(t, tree.ndb.leafNodes(), 3) +} + +func TestVersionedTreeSpecialCase(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(100, db.NewMemDB()) + + tree.Set([]byte("key1"), []byte("val0")) + tree.Set([]byte("key2"), []byte("val0")) + tree.SaveVersion(1) + + tree.Set([]byte("key1"), []byte("val1")) + tree.Set([]byte("key2"), []byte("val1")) + tree.SaveVersion(2) + + tree.Set([]byte("key2"), []byte("val2")) + tree.SaveVersion(4) + + tree.DeleteVersion(2) + + _, val := tree.GetVersioned([]byte("key2"), 1) + require.Equal("val0", string(val)) +} + +func TestVersionedTreeSpecialCase2(t *testing.T) { + require := require.New(t) + d := db.NewMemDB() + + tree := NewVersionedTree(100, d) + + tree.Set([]byte("key1"), []byte("val0")) + tree.Set([]byte("key2"), []byte("val0")) + tree.SaveVersion(1) + + tree.Set([]byte("key1"), []byte("val1")) + tree.Set([]byte("key2"), []byte("val1")) + tree.SaveVersion(2) + + tree.Set([]byte("key2"), []byte("val2")) + tree.SaveVersion(4) + + tree = NewVersionedTree(100, d) + require.NoError(tree.Load()) + + require.NoError(tree.DeleteVersion(2)) + + _, val := tree.GetVersioned([]byte("key2"), 1) + require.Equal("val0", string(val)) +} + +func TestVersionedTreeSpecialCase3(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(0, db.NewMemDB()) + + tree.Set([]byte("m"), []byte("liWT0U6G")) + tree.Set([]byte("G"), []byte("7PxRXwUA")) + tree.SaveVersion(1) + + tree.Set([]byte("7"), []byte("XRLXgf8C")) + tree.SaveVersion(2) + + tree.Set([]byte("r"), []byte("bBEmIXBU")) + tree.SaveVersion(3) + + tree.Set([]byte("i"), []byte("kkIS35te")) + tree.SaveVersion(4) + + tree.Set([]byte("k"), []byte("CpEnpzKJ")) + tree.SaveVersion(5) + + tree.DeleteVersion(1) + tree.DeleteVersion(2) + tree.DeleteVersion(3) + tree.DeleteVersion(4) + + require.Equal(tree.nodeSize(), len(tree.ndb.nodes())) +} + +func TestVersionedTreeSaveAndLoad(t *testing.T) { + require := require.New(t) + d := db.NewMemDB() + tree := NewVersionedTree(0, d) + + // Loading with an empty root is a no-op. + tree.Load() + + tree.Set([]byte("C"), []byte("so43QQFN")) + tree.SaveVersion(1) + + tree.Set([]byte("A"), []byte("ut7sTTAO")) + tree.SaveVersion(2) + + tree.Set([]byte("X"), []byte("AoWWC1kN")) + tree.SaveVersion(3) + + tree.SaveVersion(4) + tree.SaveVersion(5) + tree.SaveVersion(6) + + preHash := tree.Hash() + require.NotNil(preHash) + + require.Equal(uint64(6), tree.LatestVersion()) + + // Reload the tree, to test that roots and orphans are properly loaded. + ntree := NewVersionedTree(0, d) + ntree.Load() + + require.False(ntree.IsEmpty()) + require.Equal(uint64(6), ntree.LatestVersion()) + + postHash := ntree.Hash() + require.Equal(preHash, postHash) + + ntree.Set([]byte("T"), []byte("MhkWjkVy")) + ntree.SaveVersion(7) + + ntree.DeleteVersion(6) + ntree.DeleteVersion(5) + ntree.DeleteVersion(1) + ntree.DeleteVersion(2) + ntree.DeleteVersion(4) + ntree.DeleteVersion(3) + + require.False(ntree.IsEmpty()) + require.Equal(4, ntree.Size()) + require.Len(ntree.ndb.nodes(), ntree.nodeSize()) +} + +func TestVersionedTreeErrors(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(100, db.NewMemDB()) + + // Can't save with empty tree. + _, err := tree.SaveVersion(1) + require.Error(err) + + // Can't delete non-existent versions. + require.Error(tree.DeleteVersion(1)) + require.Error(tree.DeleteVersion(99)) + + tree.Set([]byte("key"), []byte("val")) + + // `0` is an invalid version number. + _, err = tree.SaveVersion(0) + require.Error(err) + + // Saving version `1` is ok. + _, err = tree.SaveVersion(1) + require.NoError(err) + + // Can't delete current version. + require.Error(tree.DeleteVersion(1)) + + // Trying to get a key from a version which doesn't exist. + _, val := tree.GetVersioned([]byte("key"), 404) + require.Nil(val) + + // Same thing with proof. We get an error because a proof couldn't be + // constructed. + val, proof, err := tree.GetVersionedWithProof([]byte("key"), 404) + require.Nil(val) + require.Nil(proof) + require.Error(err) +} + +func TestVersionedCheckpoints(t *testing.T) { + require := require.New(t) + d, closeDB := getTestDB() + defer closeDB() + + tree := NewVersionedTree(100, d) + versions := 50 + keysPerVersion := 10 + versionsPerCheckpoint := 5 + keys := map[uint64]([][]byte){} + + for i := 1; i <= versions; i++ { + for j := 0; j < keysPerVersion; j++ { + k := []byte(cmn.RandStr(1)) + v := []byte(cmn.RandStr(8)) + keys[uint64(i)] = append(keys[uint64(i)], k) + tree.Set(k, v) + } + tree.SaveVersion(uint64(i)) + } + + for i := 1; i <= versions; i++ { + if i%versionsPerCheckpoint != 0 { + tree.DeleteVersion(uint64(i)) + } + } + + // Make sure all keys exist at least once. + for _, ks := range keys { + for _, k := range ks { + _, val := tree.Get(k) + require.NotEmpty(val) + } + } + + // Make sure all keys from deleted versions aren't present. + for i := 1; i <= versions; i++ { + if i%versionsPerCheckpoint != 0 { + for _, k := range keys[uint64(i)] { + _, val := tree.GetVersioned(k, uint64(i)) + require.Nil(val) + } + } + } + + // Make sure all keys exist at all checkpoints. + for i := 1; i <= versions; i++ { + for _, k := range keys[uint64(i)] { + if i%versionsPerCheckpoint == 0 { + _, val := tree.GetVersioned(k, uint64(i)) + require.NotEmpty(val) + } + } + } +} + +func TestVersionedCheckpointsSpecialCase(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(0, db.NewMemDB()) + key := []byte("k") + + tree.Set(key, []byte("val1")) + + tree.SaveVersion(1) + // ... + tree.SaveVersion(10) + // ... + tree.SaveVersion(19) + // ... + // This orphans "k" at version 1. + tree.Set(key, []byte("val2")) + tree.SaveVersion(20) + + // When version 1 is deleted, the orphans should move to the next + // checkpoint, which is version 10. + tree.DeleteVersion(1) + + _, val := tree.GetVersioned(key, 10) + require.NotEmpty(val) + require.Equal([]byte("val1"), val) +} + +func TestVersionedCheckpointsSpecialCase2(t *testing.T) { + tree := NewVersionedTree(0, db.NewMemDB()) + + tree.Set([]byte("U"), []byte("XamDUtiJ")) + tree.Set([]byte("A"), []byte("UkZBuYIU")) + tree.Set([]byte("H"), []byte("7a9En4uw")) + tree.Set([]byte("V"), []byte("5HXU3pSI")) + tree.SaveVersion(1) + + tree.Set([]byte("U"), []byte("Replaced")) + tree.Set([]byte("A"), []byte("Replaced")) + tree.SaveVersion(2) + + tree.Set([]byte("X"), []byte("New")) + tree.SaveVersion(3) + + tree.DeleteVersion(1) + tree.DeleteVersion(2) +} + +func TestVersionedCheckpointsSpecialCase3(t *testing.T) { + tree := NewVersionedTree(0, db.NewMemDB()) + + tree.Set([]byte("n"), []byte("2wUCUs8q")) + tree.Set([]byte("l"), []byte("WQ7mvMbc")) + tree.SaveVersion(2) + + tree.Set([]byte("N"), []byte("ved29IqU")) + tree.Set([]byte("v"), []byte("01jquVXU")) + tree.SaveVersion(5) + + tree.Set([]byte("l"), []byte("bhIpltPM")) + tree.Set([]byte("B"), []byte("rj97IKZh")) + tree.SaveVersion(6) + + tree.DeleteVersion(5) + + tree.GetVersioned([]byte("m"), 2) +} + +func TestVersionedCheckpointsSpecialCase4(t *testing.T) { + tree := NewVersionedTree(0, db.NewMemDB()) + + tree.Set([]byte("U"), []byte("XamDUtiJ")) + tree.Set([]byte("A"), []byte("UkZBuYIU")) + tree.Set([]byte("H"), []byte("7a9En4uw")) + tree.Set([]byte("V"), []byte("5HXU3pSI")) + tree.SaveVersion(1) + + tree.Remove([]byte("U")) + tree.Remove([]byte("A")) + tree.SaveVersion(2) + + tree.Set([]byte("X"), []byte("New")) + tree.SaveVersion(3) + + _, val := tree.GetVersioned([]byte("A"), 2) + require.Nil(t, val) + + _, val = tree.GetVersioned([]byte("A"), 1) + require.NotEmpty(t, val) + + tree.DeleteVersion(1) + tree.DeleteVersion(2) + + _, val = tree.GetVersioned([]byte("A"), 2) + require.Nil(t, val) + + _, val = tree.GetVersioned([]byte("A"), 1) + require.Nil(t, val) +} + +func TestVersionedCheckpointsSpecialCase5(t *testing.T) { + tree := NewVersionedTree(0, db.NewMemDB()) + + tree.Set([]byte("R"), []byte("ygZlIzeW")) + tree.SaveVersion(1) + + tree.Set([]byte("j"), []byte("ZgmCWyo2")) + tree.SaveVersion(2) + + tree.Set([]byte("R"), []byte("vQDaoz6Z")) + tree.SaveVersion(3) + + tree.DeleteVersion(1) + + tree.GetVersioned([]byte("R"), 2) +} + +func TestVersionedCheckpointsSpecialCase6(t *testing.T) { + tree := NewVersionedTree(0, db.NewMemDB()) + + tree.Set([]byte("Y"), []byte("MW79JQeV")) + tree.Set([]byte("7"), []byte("Kp0ToUJB")) + tree.Set([]byte("Z"), []byte("I26B1jPG")) + tree.Set([]byte("6"), []byte("ZG0iXq3h")) + tree.Set([]byte("2"), []byte("WOR27LdW")) + tree.Set([]byte("4"), []byte("MKMvc6cn")) + tree.SaveVersion(1) + + tree.Set([]byte("1"), []byte("208dOu40")) + tree.Set([]byte("G"), []byte("7isI9OQH")) + tree.Set([]byte("8"), []byte("zMC1YwpH")) + tree.SaveVersion(2) + + tree.Set([]byte("7"), []byte("bn62vWbq")) + tree.Set([]byte("5"), []byte("wZuLGDkZ")) + tree.SaveVersion(3) + + tree.DeleteVersion(1) + tree.DeleteVersion(2) + + tree.GetVersioned([]byte("Y"), 1) + tree.GetVersioned([]byte("7"), 1) + tree.GetVersioned([]byte("Z"), 1) + tree.GetVersioned([]byte("6"), 1) + tree.GetVersioned([]byte("s"), 1) + tree.GetVersioned([]byte("2"), 1) + tree.GetVersioned([]byte("4"), 1) +} + +func TestVersionedCheckpointsSpecialCase7(t *testing.T) { + tree := NewVersionedTree(100, db.NewMemDB()) + + tree.Set([]byte("n"), []byte("OtqD3nyn")) + tree.Set([]byte("W"), []byte("kMdhJjF5")) + tree.Set([]byte("A"), []byte("BM3BnrIb")) + tree.Set([]byte("I"), []byte("QvtCH970")) + tree.Set([]byte("L"), []byte("txKgOTqD")) + tree.Set([]byte("Y"), []byte("NAl7PC5L")) + tree.SaveVersion(1) + + tree.Set([]byte("7"), []byte("qWcEAlyX")) + tree.SaveVersion(2) + + tree.Set([]byte("M"), []byte("HdQwzA64")) + tree.Set([]byte("3"), []byte("2Naa77fo")) + tree.Set([]byte("A"), []byte("SRuwKOTm")) + tree.Set([]byte("I"), []byte("oMX4aAOy")) + tree.Set([]byte("4"), []byte("dKfvbEOc")) + tree.SaveVersion(3) + + tree.Set([]byte("D"), []byte("3U4QbXCC")) + tree.Set([]byte("B"), []byte("FxExhiDq")) + tree.SaveVersion(5) + + tree.Set([]byte("A"), []byte("tWQgbFCY")) + tree.SaveVersion(6) + + tree.DeleteVersion(5) + + tree.GetVersioned([]byte("A"), 3) +} + +func TestVersionedTreeEfficiency(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(0, db.NewMemDB()) + versions := 20 + keysPerVersion := 100 + keysAddedPerVersion := map[int]int{} + + keysAdded := 0 + for i := 1; i <= versions; i++ { + for j := 0; j < keysPerVersion; j++ { + // Keys of size one are likely to be overwritten. + tree.Set([]byte(cmn.RandStr(1)), []byte(cmn.RandStr(8))) + } + sizeBefore := len(tree.ndb.nodes()) + tree.SaveVersion(uint64(i)) + sizeAfter := len(tree.ndb.nodes()) + change := sizeAfter - sizeBefore + keysAddedPerVersion[i] = change + keysAdded += change + } + + keysDeleted := 0 + for i := 1; i < versions; i++ { + sizeBefore := len(tree.ndb.nodes()) + tree.DeleteVersion(uint64(i)) + sizeAfter := len(tree.ndb.nodes()) + + change := sizeBefore - sizeAfter + keysDeleted += change + + require.InDelta(change, keysAddedPerVersion[i], float64(keysPerVersion)/5) + } + require.Equal(keysAdded-tree.nodeSize(), keysDeleted) +} + +func TestVersionedTreeProofs(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(0, db.NewMemDB()) + + tree.Set([]byte("k1"), []byte("v1")) + tree.Set([]byte("k2"), []byte("v1")) + tree.Set([]byte("k3"), []byte("v1")) + tree.SaveVersion(1) + + root1 := tree.Hash() + + tree.Set([]byte("k2"), []byte("v2")) + tree.Set([]byte("k4"), []byte("v2")) + tree.SaveVersion(2) + + root2 := tree.Hash() + require.NotEqual(root1, root2) + + tree.Remove([]byte("k2")) + tree.SaveVersion(3) + + root3 := tree.Hash() + require.NotEqual(root2, root3) + + val, proof, err := tree.GetVersionedWithProof([]byte("k2"), 1) + require.NoError(err) + require.EqualValues(val, []byte("v1")) + require.EqualValues(1, proof.(*KeyExistsProof).Version) + require.NoError(proof.Verify([]byte("k2"), val, root1)) + + val, proof, err = tree.GetVersionedWithProof([]byte("k4"), 1) + require.NoError(err) + require.Nil(val) + require.NoError(proof.Verify([]byte("k4"), nil, root1)) + require.Error(proof.Verify([]byte("k4"), val, root2)) + require.EqualValues(1, proof.(*KeyAbsentProof).Version) + + val, proof, err = tree.GetVersionedWithProof([]byte("k2"), 2) + require.NoError(err) + require.EqualValues(val, []byte("v2")) + require.NoError(proof.Verify([]byte("k2"), val, root2)) + require.Error(proof.Verify([]byte("k2"), val, root1)) + require.EqualValues(2, proof.(*KeyExistsProof).Version) + + val, proof, err = tree.GetVersionedWithProof([]byte("k1"), 2) + require.NoError(err) + require.EqualValues(val, []byte("v1")) + require.NoError(proof.Verify([]byte("k1"), val, root2)) + require.EqualValues(1, proof.(*KeyExistsProof).Version) // Key version = 1 + + val, proof, err = tree.GetVersionedWithProof([]byte("k2"), 3) + require.NoError(err) + require.Nil(val) + require.NoError(proof.Verify([]byte("k2"), nil, root3)) + require.Error(proof.Verify([]byte("k2"), nil, root1)) + require.Error(proof.Verify([]byte("k2"), nil, root2)) + require.EqualValues(1, proof.(*KeyAbsentProof).Version) +} + +func TestVersionedTreeHash(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(0, db.NewMemDB()) + + require.Nil(tree.Hash()) + tree.Set([]byte("I"), []byte("D")) + require.Nil(tree.Hash()) + + hash1, _ := tree.SaveVersion(1) + + tree.Set([]byte("I"), []byte("F")) + require.EqualValues(hash1, tree.Hash()) + + hash2, _ := tree.SaveVersion(2) + + val, proof, err := tree.GetVersionedWithProof([]byte("I"), 2) + require.NoError(err) + require.EqualValues(val, []byte("F")) + require.NoError(proof.Verify([]byte("I"), val, hash2)) +} + +func TestNilValueSemantics(t *testing.T) { + require := require.New(t) + tree := NewVersionedTree(0, db.NewMemDB()) + + require.Panics(func() { + tree.Set([]byte("k"), nil) + }) +} + +func TestCopyValueSemantics(t *testing.T) { + require := require.New(t) + + tree := NewVersionedTree(0, db.NewMemDB()) + + val := []byte("v1") + + tree.Set([]byte("k"), val) + _, v := tree.Get([]byte("k")) + require.Equal([]byte("v1"), v) + + val[1] = '2' + + _, val = tree.Get([]byte("k")) + require.Equal([]byte("v2"), val) +} + +//////////////////////////// BENCHMARKS /////////////////////////////////////// + +func BenchmarkTreeLoadAndDelete(b *testing.B) { + numVersions := 5000 + numKeysPerVersion := 10 + + d, err := db.NewGoLevelDB("bench", ".") + if err != nil { + panic(err) + } + defer d.Close() + defer os.RemoveAll("./bench.db") + + tree := NewVersionedTree(0, d) + for v := 1; v < numVersions; v++ { + for i := 0; i < numKeysPerVersion; i++ { + tree.Set([]byte(cmn.RandStr(16)), cmn.RandBytes(32)) + } + tree.SaveVersion(uint64(v)) + } + + b.Run("LoadAndDelete", func(b *testing.B) { + for n := 0; n < b.N; n++ { + b.StopTimer() + tree = NewVersionedTree(0, d) + runtime.GC() + b.StartTimer() + + // Load the tree from disk. + tree.Load() + + // Delete about 10% of the versions randomly. + // The trade-off is usually between load efficiency and delete + // efficiency, which is why we do both in this benchmark. + // If we can load quickly into a data-structure that allows for + // efficient deletes, we are golden. + for v := 0; v < numVersions/10; v++ { + version := (cmn.RandInt() % numVersions) + 1 + tree.DeleteVersion(uint64(version)) + } + } + }) +} diff --git a/iavl_util.go b/util.go similarity index 61% rename from iavl_util.go rename to util.go index add1c3c23..b28b4877f 100644 --- a/iavl_util.go +++ b/util.go @@ -4,23 +4,14 @@ import ( "fmt" ) -// Prints the in-memory children recursively. -func PrintIAVLNode(node *IAVLNode) { - fmt.Println("==== NODE") - if node != nil { - printIAVLNode(node, 0) - } - fmt.Println("==== END") -} - -func printIAVLNode(node *IAVLNode, indent int) { +func printNode(node *Node, indent int) { indentPrefix := "" for i := 0; i < indent; i++ { indentPrefix += " " } if node.rightNode != nil { - printIAVLNode(node.rightNode, indent+1) + printNode(node.rightNode, indent+1) } else if node.rightHash != nil { fmt.Printf("%s %X\n", indentPrefix, node.rightHash) } @@ -28,7 +19,7 @@ func printIAVLNode(node *IAVLNode, indent int) { fmt.Printf("%s%v:%v\n", indentPrefix, node.key, node.height) if node.leftNode != nil { - printIAVLNode(node.leftNode, indent+1) + printNode(node.leftNode, indent+1) } else if node.leftHash != nil { fmt.Printf("%s %X\n", indentPrefix, node.leftHash) } diff --git a/versioned_tree.go b/versioned_tree.go new file mode 100644 index 000000000..607b10c0f --- /dev/null +++ b/versioned_tree.go @@ -0,0 +1,207 @@ +package iavl + +import ( + "fmt" + + "github.com/pkg/errors" + dbm "github.com/tendermint/tmlibs/db" +) + +var ErrVersionDoesNotExist = fmt.Errorf("version does not exist") + +// VersionedTree is a persistent tree which keeps track of versions. +type VersionedTree struct { + *orphaningTree // The current, working tree. + versions map[uint64]*Tree // The previous, saved versions of the tree. + latestVersion uint64 // The latest saved version. + ndb *nodeDB +} + +// NewVersionedTree returns a new tree with the specified cache size and datastore. +func NewVersionedTree(cacheSize int, db dbm.DB) *VersionedTree { + ndb := newNodeDB(cacheSize, db) + head := &Tree{ndb: ndb} + + return &VersionedTree{ + orphaningTree: newOrphaningTree(head), + versions: map[uint64]*Tree{}, + ndb: ndb, + } +} + +// LatestVersion returns the latest saved version of the tree. +func (tree *VersionedTree) LatestVersion() uint64 { + return tree.latestVersion +} + +// IsEmpty returns whether or not the tree has any keys. Only trees that are +// not empty can be saved. +func (tree *VersionedTree) IsEmpty() bool { + return tree.orphaningTree.Size() == 0 +} + +// VersionExists returns whether or not a version exists. +func (tree *VersionedTree) VersionExists(version uint64) bool { + _, ok := tree.versions[version] + return ok +} + +// Tree returns the current working tree. +func (tree *VersionedTree) Tree() *Tree { + return tree.orphaningTree.Tree +} + +// Hash returns the hash of the latest saved version of the tree, as returned +// by SaveVersion. If no versions have been saved, Hash returns nil. +func (tree *VersionedTree) Hash() []byte { + if tree.latestVersion > 0 { + return tree.versions[tree.latestVersion].Hash() + } + return nil +} + +// String returns a string representation of the tree. +func (tree *VersionedTree) String() string { + return tree.ndb.String() +} + +// Set sets a key in the working tree. Nil values are not supported. +func (tree *VersionedTree) Set(key, val []byte) bool { + return tree.orphaningTree.Set(key, val) +} + +// Remove removes a key from the working tree. +func (tree *VersionedTree) Remove(key []byte) ([]byte, bool) { + return tree.orphaningTree.Remove(key) +} + +// Load a versioned tree from disk. All tree versions are loaded automatically. +func (tree *VersionedTree) Load() error { + roots, err := tree.ndb.getRoots() + if err != nil { + return err + } + if len(roots) == 0 { + return nil + } + + // Load all roots from the database. + for version, root := range roots { + t := &Tree{ndb: tree.ndb} + t.load(root) + + tree.versions[version] = t + + if version > tree.latestVersion { + tree.latestVersion = version + } + } + + // Set the working tree to a copy of the latest. + tree.orphaningTree = newOrphaningTree( + tree.versions[tree.latestVersion].clone(), + ) + + return nil +} + +// GetVersioned gets the value at the specified key and version. +func (tree *VersionedTree) GetVersioned(key []byte, version uint64) ( + index int, value []byte, +) { + if t, ok := tree.versions[version]; ok { + return t.Get(key) + } + return -1, nil +} + +// SaveVersion saves a new tree version to disk, based on the current state of +// the tree. Multiple calls to SaveVersion with the same version are not allowed. +func (tree *VersionedTree) SaveVersion(version uint64) ([]byte, error) { + if _, ok := tree.versions[version]; ok { + return nil, errors.Errorf("version %d was already saved", version) + } + if tree.root == nil { + return nil, ErrNilRoot + } + if version == 0 { + return nil, errors.New("version must be greater than zero") + } + if version <= tree.latestVersion { + return nil, errors.Errorf("version must be greater than latest (%d <= %d)", + version, tree.latestVersion) + } + + tree.latestVersion = version + tree.versions[version] = tree.orphaningTree.Tree + + tree.orphaningTree.SaveVersion(version) + tree.orphaningTree = newOrphaningTree( + tree.versions[version].clone(), + ) + + tree.ndb.SaveRoot(tree.root, version) + tree.ndb.Commit() + + return tree.root.hash, nil +} + +// DeleteVersion deletes a tree version from disk. The version can then no +// longer be accessed. +func (tree *VersionedTree) DeleteVersion(version uint64) error { + if version == 0 { + return errors.New("version must be greater than 0") + } + if version == tree.latestVersion { + return errors.Errorf("cannot delete latest saved version (%d)", version) + } + if _, ok := tree.versions[version]; !ok { + return errors.WithStack(ErrVersionDoesNotExist) + } + + tree.ndb.DeleteVersion(version) + tree.ndb.Commit() + + delete(tree.versions, version) + + return nil +} + +// GetVersionedWithProof gets the value under the key at the specified version +// if it exists, or returns nil. A proof of existence or absence is returned +// alongside the value. +func (tree *VersionedTree) GetVersionedWithProof(key []byte, version uint64) ([]byte, KeyProof, error) { + if t, ok := tree.versions[version]; ok { + return t.GetWithProof(key) + } + return nil, nil, errors.WithStack(ErrVersionDoesNotExist) +} + +// GetVersionedRangeWithProof gets key/value pairs within the specified range +// and limit. To specify a descending range, swap the start and end keys. +// +// Returns a list of keys, a list of values and a proof. +func (tree *VersionedTree) GetVersionedRangeWithProof(startKey, endKey []byte, limit int, version uint64) ([][]byte, [][]byte, *KeyRangeProof, error) { + if t, ok := tree.versions[version]; ok { + return t.GetRangeWithProof(startKey, endKey, limit) + } + return nil, nil, nil, errors.WithStack(ErrVersionDoesNotExist) +} + +// GetVersionedFirstInRangeWithProof gets the first key/value pair in the +// specified range, with a proof. +func (tree *VersionedTree) GetVersionedFirstInRangeWithProof(startKey, endKey []byte, version uint64) ([]byte, []byte, *KeyFirstInRangeProof, error) { + if t, ok := tree.versions[version]; ok { + return t.GetFirstInRangeWithProof(startKey, endKey) + } + return nil, nil, nil, errors.WithStack(ErrVersionDoesNotExist) +} + +// GetVersionedLastInRangeWithProof gets the last key/value pair in the +// specified range, with a proof. +func (tree *VersionedTree) GetVersionedLastInRangeWithProof(startKey, endKey []byte, version uint64) ([]byte, []byte, *KeyLastInRangeProof, error) { + if t, ok := tree.versions[version]; ok { + return t.GetLastInRangeWithProof(startKey, endKey) + } + return nil, nil, nil, errors.WithStack(ErrVersionDoesNotExist) +} diff --git a/iavl_with_gcc_test.go b/with_gcc_test.go similarity index 100% rename from iavl_with_gcc_test.go rename to with_gcc_test.go