Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Mutable/Immutable refactor and GetImmutable snapshots #92

Merged
merged 14 commits into from
Aug 11, 2018
Merged

Mutable/Immutable refactor and GetImmutable snapshots #92

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
206b7ac
Move orphaningTree logic into VersionedTree
UnitylChaos Jul 24, 2018
cbab68b
Move Tree.Set and Node.set to VersionedTree, fix tests
UnitylChaos Jul 24, 2018
68ede19
Move Tree.Remove and node.remove to VersionedTree
UnitylChaos Jul 24, 2018
13ba7f5
Rename VersionedTree/Tree to MutableTree/ImmutableTree
UnitylChaos Jul 24, 2018
442664c
Rename files to match type names
UnitylChaos Jul 24, 2018
7c71f0e
Fix lints
UnitylChaos Jul 24, 2018
68c7d38
Add GetImmutable and lazy loading of MutableTree
UnitylChaos Jul 25, 2018
36b7550
Move balance and rotate from Node to MutableTree
UnitylChaos Jul 26, 2018
badf974
Add benchmarks with -benchmem, remove incomplete benchmark
UnitylChaos Jul 31, 2018
f1657ab
Rename variables in rotation functions
UnitylChaos Aug 2, 2018
8ad44f7
Add test to check for consistent tracking of orphans
UnitylChaos Aug 2, 2018
3d7299f
Add back ability to load old versions with idempotent saves
UnitylChaos Aug 3, 2018
4a99aa5
Add idempotent save test case
UnitylChaos Aug 6, 2018
bab6ea5
Fix capitalization of error message
UnitylChaos Aug 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 17 additions & 18 deletions basic_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
)

func TestBasic(t *testing.T) {
tree := NewTree(nil, 0)
tree := NewMutableTree(db.NewMemDB(), 0)
up := tree.Set([]byte("1"), []byte("one"))
if up {
t.Error("Did not expect an update (should have been create)")
Expand Down Expand Up @@ -103,7 +103,7 @@ func TestBasic(t *testing.T) {

func TestUnit(t *testing.T) {

expectHash := func(tree *Tree, hashCount int64) {
expectHash := func(tree *ImmutableTree, hashCount int64) {
// ensure number of new hash calculations is as expected.
hash, count := tree.hashWithCount()
if count != hashCount {
Expand All @@ -121,7 +121,7 @@ func TestUnit(t *testing.T) {
}
}

expectSet := func(tree *Tree, i int, repr string, hashCount int64) {
expectSet := func(tree *MutableTree, i int, repr string, hashCount int64) {
origNode := tree.root
updated := tree.Set(i2b(i), []byte{})
// ensure node was added & structure is as expected.
Expand All @@ -130,11 +130,11 @@ func TestUnit(t *testing.T) {
i, P(origNode), repr, P(tree.root), updated)
}
// ensure hash calculation requirements
expectHash(tree, hashCount)
expectHash(tree.ImmutableTree, hashCount)
tree.root = origNode
}

expectRemove := func(tree *Tree, i int, repr string, hashCount int64) {
expectRemove := func(tree *MutableTree, i int, repr string, hashCount int64) {
origNode := tree.root
value, removed := tree.Remove(i2b(i))
// ensure node was added & structure is as expected.
Expand All @@ -143,7 +143,7 @@ func TestUnit(t *testing.T) {
i, P(origNode), repr, P(tree.root), value, removed)
}
// ensure hash calculation requirements
expectHash(tree, hashCount)
expectHash(tree.ImmutableTree, hashCount)
tree.root = origNode
}

Expand Down Expand Up @@ -190,7 +190,7 @@ func TestRemove(t *testing.T) {

d := db.NewDB("test", "memdb", "")
defer d.Close()
t1 := NewVersionedTree(d, size)
t1 := NewMutableTree(d, size)

// insert a bunch of random nodes
keys := make([][]byte, size)
Expand Down Expand Up @@ -220,7 +220,7 @@ func TestIntegration(t *testing.T) {
}

records := make([]*record, 400)
tree := NewTree(nil, 0)
tree := NewMutableTree(db.NewMemDB(), 0)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why did this change?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Posting what I said in the call here for future reference:

Anywhere in the tests where we were using Tree mutably I had to change it to MutableTree, and if you pass nil as the db to MutableTree it errors when constructing the nodedb. Using MutableTree with a MemDB as nodedb is equivalent to the old code using Tree.


randomRecord := func() *record {
return &record{randstr(20), randstr(20)}
Expand Down Expand Up @@ -302,7 +302,7 @@ func TestIterateRange(t *testing.T) {
}
sort.Strings(keys)

tree := NewTree(nil, 0)
tree := NewMutableTree(db.NewMemDB(), 0)

// insert all the data
for _, r := range records {
Expand Down Expand Up @@ -372,14 +372,14 @@ func TestPersistence(t *testing.T) {
}

// Construct some tree and save it
t1 := NewVersionedTree(db, 0)
t1 := NewMutableTree(db, 0)
for key, value := range records {
t1.Set([]byte(key), []byte(value))
}
t1.SaveVersion()

// Load a tree
t2 := NewVersionedTree(db, 0)
t2 := NewMutableTree(db, 0)
t2.Load()
for key, value := range records {
_, t2value := t2.Get64([]byte(key))
Expand All @@ -390,12 +390,11 @@ func TestPersistence(t *testing.T) {
}

func TestProof(t *testing.T) {
t.Skipf("This test has a race condition causing it to occasionally panic.")

// Construct some random tree
db := db.NewMemDB()
tree := NewVersionedTree(db, 100)
for i := 0; i < 1000; i++ {
tree := NewMutableTree(db, 100)
for i := 0; i < 10; i++ {
key, value := randstr(20), randstr(20)
tree.Set([]byte(key), []byte(value))
}
Expand All @@ -404,7 +403,7 @@ func TestProof(t *testing.T) {
tree.SaveVersion()

// Add more items so it's not all persisted
for i := 0; i < 100; i++ {
for i := 0; i < 10; i++ {
key, value := randstr(20), randstr(20)
tree.Set([]byte(key), []byte(value))
}
Expand All @@ -415,15 +414,15 @@ func TestProof(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, value, value2)
if assert.NotNil(t, proof) {
verifyProof(t, proof, tree.Hash())
verifyProof(t, proof, tree.WorkingHash())
}
return false
})
}

func TestTreeProof(t *testing.T) {
db := db.NewMemDB()
tree := NewTree(db, 100)
tree := NewMutableTree(db, 100)

// should get false for proof with nil root
_, _, err := tree.GetWithProof([]byte("foo"))
Expand All @@ -442,7 +441,7 @@ func TestTreeProof(t *testing.T) {
assert.NoError(t, err)

// valid proof for real keys
root := tree.Hash()
root := tree.WorkingHash()
for _, key := range keys {
value, proof, err := tree.GetWithProof(key)
if assert.NoError(t, err) {
Expand Down
18 changes: 9 additions & 9 deletions benchmarks/bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ func randBytes(length int) []byte {
return key
}

func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.VersionedTree, [][]byte) {
t := iavl.NewVersionedTree(db, size)
func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.MutableTree, [][]byte) {
t := iavl.NewMutableTree(db, size)
keys := make([][]byte, size)

for i := 0; i < size; i++ {
Expand All @@ -35,7 +35,7 @@ func prepareTree(b *testing.B, db db.DB, size, keyLen, dataLen int) (*iavl.Versi
}

// commit tree saves a new version and deletes and old one...
func commitTree(b *testing.B, t *iavl.VersionedTree) {
func commitTree(b *testing.B, t *iavl.MutableTree) {
t.Hash()
_, version, err := t.SaveVersion()
if err != nil {
Expand All @@ -49,22 +49,22 @@ func commitTree(b *testing.B, t *iavl.VersionedTree) {
}
}

func runQueries(b *testing.B, t *iavl.VersionedTree, keyLen int) {
func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) {
for i := 0; i < b.N; i++ {
q := randBytes(keyLen)
t.Get(q)
}
}

func runKnownQueries(b *testing.B, t *iavl.VersionedTree, keys [][]byte) {
func runKnownQueries(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
l := int32(len(keys))
for i := 0; i < b.N; i++ {
q := keys[rand.Int31n(l)]
t.Get(q)
}
}

func runInsert(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize int) *iavl.VersionedTree {
func runInsert(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int) *iavl.MutableTree {
for i := 1; i <= b.N; i++ {
t.Set(randBytes(keyLen), randBytes(dataLen))
if i%blockSize == 0 {
Expand All @@ -75,7 +75,7 @@ func runInsert(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize i
return t
}

func runUpdate(b *testing.B, t *iavl.VersionedTree, dataLen, blockSize int, keys [][]byte) *iavl.VersionedTree {
func runUpdate(b *testing.B, t *iavl.MutableTree, dataLen, blockSize int, keys [][]byte) *iavl.MutableTree {
l := int32(len(keys))
for i := 1; i <= b.N; i++ {
key := keys[rand.Int31n(l)]
Expand All @@ -87,7 +87,7 @@ func runUpdate(b *testing.B, t *iavl.VersionedTree, dataLen, blockSize int, keys
return t
}

func runDelete(b *testing.B, t *iavl.VersionedTree, blockSize int, keys [][]byte) *iavl.VersionedTree {
func runDelete(b *testing.B, t *iavl.MutableTree, blockSize int, keys [][]byte) *iavl.MutableTree {
var key []byte
l := int32(len(keys))
for i := 1; i <= b.N; i++ {
Expand All @@ -103,7 +103,7 @@ func runDelete(b *testing.B, t *iavl.VersionedTree, blockSize int, keys [][]byte
}

// runBlock measures time for an entire block, not just one tx
func runBlock(b *testing.B, t *iavl.VersionedTree, keyLen, dataLen, blockSize int, keys [][]byte) *iavl.VersionedTree {
func runBlock(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int, keys [][]byte) *iavl.MutableTree {
l := int32(len(keys))

// XXX: This was adapted to work with VersionedTree but needs to be re-thought.
Expand Down
57 changes: 0 additions & 57 deletions benchmarks/results/digial-ocean-32gb-18.04-a3d16d8.txt
View file
Open in desktop

This file was deleted.

Loading