feat: Fast storage optimization for queries and iterations

.gitignore

@@ -12,3 +12,6 @@ cpu*.out
 mem*.out
 cpu*.pdf
 mem*.pdf
+
+# IDE files
+.idea/*

CHANGELOG.md

@@ -4,6 +4,12 @@
 
 ### Improvements
 
+- [\#468](https://github.com/cosmos/iavl/pull/468) Fast storage optimization for queries and iterations
+
+## 0.17.3 (December 1, 2021)
+
+### Improvements
+
 - [\#445](https://github.com/cosmos/iavl/pull/445) Bump github.com/tendermint/tendermint to v0.35.0
 
 ### Bug Fixes

basic_test.go

@@ -35,72 +35,127 @@ func TestBasic(t *testing.T) {
 
 	// Test 0x00
 	{
-		idx, val := tree.Get([]byte{0x00})
+		key := []byte{0x00}
+		expected := ""
+
+		idx, val := tree.GetWithIndex(key)
 		if val != nil {
 			t.Errorf("Expected no value to exist")
 		}
 		if idx != 0 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "" {
+		if string(val) != expected {
 			t.Errorf("Unexpected value %v", string(val))
 		}
+
+		val = tree.Get(key)
+		if val != nil {
+			t.Errorf("Fast method - expected no value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %v", string(val))
+		}
 	}
 
 	// Test "1"
 	{
-		idx, val := tree.Get([]byte("1"))
+		key := []byte("1")
+		expected := "one"
+
+		idx, val := tree.GetWithIndex(key)
 		if val == nil {
 			t.Errorf("Expected value to exist")
 		}
 		if idx != 0 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "one" {
+		if string(val) != expected {
 			t.Errorf("Unexpected value %v", string(val))
 		}
+
+		val = tree.Get(key)
+		if val == nil {
+			t.Errorf("Fast method - expected value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %v", string(val))
+		}
 	}
 
 	// Test "2"
 	{
-		idx, val := tree.Get([]byte("2"))
+		key := []byte("2")
+		expected := "TWO"
+
+		idx, val := tree.GetWithIndex(key)
 		if val == nil {
 			t.Errorf("Expected value to exist")
 		}
 		if idx != 1 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "TWO" {
+		if string(val) != expected {
 			t.Errorf("Unexpected value %v", string(val))
 		}
+
+		val = tree.Get(key)
+		if val == nil {
+			t.Errorf("Fast method - expected value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %v", string(val))
+		}
 	}
 
 	// Test "4"
 	{
-		idx, val := tree.Get([]byte("4"))
+		key := []byte("4")
+		expected := ""
+
+		idx, val := tree.GetWithIndex(key)
 		if val != nil {
 			t.Errorf("Expected no value to exist")
 		}
 		if idx != 2 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "" {
+		if string(val) != expected {
 			t.Errorf("Unexpected value %v", string(val))
 		}
+
+		val = tree.Get(key)
+		if val != nil {
+			t.Errorf("Fast method - expected no value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %v", string(val))
+		}
 	}
 
 	// Test "6"
 	{
-		idx, val := tree.Get([]byte("6"))
+		key := []byte("6")
+		expected := ""
+
+		idx, val := tree.GetWithIndex(key)
 		if val != nil {
 			t.Errorf("Expected no value to exist")
 		}
 		if idx != 3 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "" {
+		if string(val) != expected {
 			t.Errorf("Unexpected value %v", string(val))
 		}
+
+		val = tree.Get(key)
+		if val != nil {
+			t.Errorf("Fast method - expected no value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %v", string(val))
+		}
 	}
 }
 
@@ -252,7 +307,7 @@ func TestIntegration(t *testing.T) {
 		if has := tree.Has([]byte(randstr(12))); has {
 			t.Error("Table has extra key")
 		}
-		if _, val := tree.Get([]byte(r.key)); string(val) != r.value {
+		if val := tree.Get([]byte(r.key)); string(val) != r.value {
 			t.Error("wrong value")
 		}
 	}
@@ -270,7 +325,7 @@ func TestIntegration(t *testing.T) {
 			if has := tree.Has([]byte(randstr(12))); has {
 				t.Error("Table has extra key")
 			}
-			_, val := tree.Get([]byte(r.key))
+			val := tree.Get([]byte(r.key))
 			if string(val) != r.value {
 				t.Error("wrong value")
 			}
@@ -388,7 +443,7 @@ func TestPersistence(t *testing.T) {
 	require.NoError(t, err)
 	t2.Load()
 	for key, value := range records {
-		_, t2value := t2.Get([]byte(key))
+		t2value := t2.Get([]byte(key))
 		if string(t2value) != value {
 			t.Fatalf("Invalid value. Expected %v, got %v", value, t2value)
 		}

benchmarks/bench_test.go

@@ -57,21 +57,93 @@ func commitTree(b *testing.B, t *iavl.MutableTree) {
 	}
 }
 
-func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) {
+// queries random keys against live state. Keys are almost certainly not in the tree.
+func runQueriesFast(b *testing.B, t *iavl.MutableTree, keyLen int) {
+	require.True(b, t.IsFastCacheEnabled())
 	for i := 0; i < b.N; i++ {
 		q := randBytes(keyLen)
 		t.Get(q)
 	}
 }
 
-func runKnownQueries(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
+// queries keys that are known to be in state
+func runKnownQueriesFast(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
+	require.True(b, t.IsFastCacheEnabled()) // to ensure fast storage is enabled
 	l := int32(len(keys))
 	for i := 0; i < b.N; i++ {
 		q := keys[rand.Int31n(l)]
 		t.Get(q)
 	}
 }
 
+func runQueriesSlow(b *testing.B, t *iavl.MutableTree, keyLen int) {
+	b.StopTimer()
+	// Save version to get an old immutable tree to query against,
+	// Fast storage is not enabled on old tree versions, allowing us to bench the desired behavior.
+	_, version, err := t.SaveVersion()
+	require.NoError(b, err)
+
+	itree, err := t.GetImmutable(version - 1)
+	require.NoError(b, err)
+	require.False(b, itree.IsFastCacheEnabled()) // to ensure fast storage is not enabled
+
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		q := randBytes(keyLen)
+		itree.GetWithIndex(q)
+	}
+}
+
+func runKnownQueriesSlow(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
+	b.StopTimer()
+	// Save version to get an old immutable tree to query against,
+	// Fast storage is not enabled on old tree versions, allowing us to bench the desired behavior.
+	_, version, err := t.SaveVersion()
+	require.NoError(b, err)
+
+	itree, err := t.GetImmutable(version - 1)
+	require.NoError(b, err)
+	require.False(b, itree.IsFastCacheEnabled()) // to ensure fast storage is not enabled
+	b.StartTimer()
+	l := int32(len(keys))
+	for i := 0; i < b.N; i++ {
+		q := keys[rand.Int31n(l)]
+		itree.GetWithIndex(q)
+	}
+}
+
+func runIterationFast(b *testing.B, t *iavl.MutableTree, expectedSize int) {
+	require.True(b, t.IsFastCacheEnabled()) // to ensure fast storage is enabled
+	for i := 0; i < b.N; i++ {
+		itr := t.ImmutableTree.Iterator(nil, nil, false)
+		iterate(b, itr, expectedSize)
+		itr.Close()
+	}
+}
+
+func runIterationSlow(b *testing.B, t *iavl.MutableTree, expectedSize int) {
+	for i := 0; i < b.N; i++ {
+		itr := iavl.NewIterator(nil, nil, false, t.ImmutableTree) // create slow iterator directly
+		iterate(b, itr, expectedSize)
+		itr.Close()
+	}
+}
+
+func iterate(b *testing.B, itr db.Iterator, expectedSize int) {
+	b.StartTimer()
+	keyValuePairs := make([][][]byte, 0, expectedSize)
+	for i := 0; i < expectedSize && itr.Valid(); i++ {
+		itr.Next()
+		keyValuePairs = append(keyValuePairs, [][]byte{itr.Key(), itr.Value()})
+	}
+	b.StopTimer()
+	if len(keyValuePairs) != expectedSize {
+		b.Errorf("iteration count mismatch: %d != %d", len(keyValuePairs), expectedSize)
+	} else {
+		b.Logf("completed %d iterations", len(keyValuePairs))
+	}
+}
+
 // func runInsert(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int) *iavl.MutableTree {
 // 	for i := 1; i <= b.N; i++ {
 // 		t.Set(randBytes(keyLen), randBytes(dataLen))
@@ -132,7 +204,7 @@ func runBlock(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int,
 			data := randBytes(dataLen)
 
 			// perform query and write on check and then real
-			// check.Get(key)
+			// check.GetFast(key)
 			// check.Set(key, data)
 			real.Get(key)
 			real.Set(key, data)
@@ -287,14 +359,33 @@ func runSuite(b *testing.B, d db.DB, initSize, blockSize, keyLen, dataLen int) {
 
 	b.ResetTimer()
 
-	b.Run("query-miss", func(sub *testing.B) {
+	b.Run("query-no-in-tree-guarantee-fast", func(sub *testing.B) {
+		sub.ReportAllocs()
+		runQueriesFast(sub, t, keyLen)
+	})
+	b.Run("query-no-in-tree-guarantee-slow", func(sub *testing.B) {
+		sub.ReportAllocs()
+		runQueriesSlow(sub, t, keyLen)
+	})
+	//
+	b.Run("query-hits-fast", func(sub *testing.B) {
+		sub.ReportAllocs()
+		runKnownQueriesFast(sub, t, keys)
+	})
+	b.Run("query-hits-slow", func(sub *testing.B) {
+		sub.ReportAllocs()
+		runKnownQueriesSlow(sub, t, keys)
+	})
+	//
+	b.Run("iteration-fast", func(sub *testing.B) {
 		sub.ReportAllocs()
-		runQueries(sub, t, keyLen)
+		runIterationFast(sub, t, initSize)
 	})
-	b.Run("query-hits", func(sub *testing.B) {
+	b.Run("iteration-slow", func(sub *testing.B) {
 		sub.ReportAllocs()
-		runKnownQueries(sub, t, keys)
+		runIterationSlow(sub, t, initSize)
 	})
+	//
 	b.Run("update", func(sub *testing.B) {
 		sub.ReportAllocs()
 		t = runUpdate(sub, t, dataLen, blockSize, keys)

export_test.go

@@ -52,8 +52,8 @@ func setupExportTreeRandom(t *testing.T) *ImmutableTree {
 		keySize   = 16
 		valueSize = 16
 
-		versions    = 32   // number of versions to generate
-		versionOps  = 4096 // number of operations (create/update/delete) per version
+		versions    = 8    // number of versions to generate
+		versionOps  = 1024 // number of operations (create/update/delete) per version
 		updateRatio = 0.4  // ratio of updates out of all operations
 		deleteRatio = 0.2  // ratio of deletes out of all operations
 	)
@@ -211,8 +211,8 @@ func TestExporter_Import(t *testing.T) {
 			require.Equal(t, tree.Version(), newTree.Version(), "Tree version mismatch")
 
 			tree.Iterate(func(key, value []byte) bool {
-				index, _ := tree.Get(key)
-				newIndex, newValue := newTree.Get(key)
+				index, _ := tree.GetWithIndex(key)
+				newIndex, newValue := newTree.GetWithIndex(key)
 				require.Equal(t, index, newIndex, "Index mismatch for key %v", key)
 				require.Equal(t, value, newValue, "Value mismatch for key %v", key)
 				return false