feat: Fast storage optimization for queries and iterations

.gitignore

@@ -12,3 +12,6 @@ cpu*.out
 mem*.out
 cpu*.pdf
 mem*.pdf
+
+# IDE files
+.idea/*

CHANGELOG.md

@@ -9,6 +9,12 @@
 
 ### Improvements
 
+- [\#468](https://github.com/cosmos/iavl/pull/468) Fast storage optimization for queries and iterations
+
+## 0.17.3 (December 1, 2021)
+
+### Improvements
+
 - [\#445](https://github.com/cosmos/iavl/pull/445) Bump github.com/tendermint/tendermint to v0.35.0
 - [\#452](https://github.com/cosmos/iavl/pull/452) Optimization: remove unnecessary (*bytes.Buffer).Reset right after creating buffer.
 - [\#453](https://github.com/cosmos/iavl/pull/453),[\#456](https://github.com/cosmos/iavl/pull/456) Optimization: buffer reuse

basic_test.go

@@ -35,71 +35,126 @@ func TestBasic(t *testing.T) {
 
 	// Test 0x00
 	{
-		idx, val := tree.Get([]byte{0x00})
+		key := []byte{0x00}
+		expected := ""
+
+		idx, val := tree.GetWithIndex(key)
 		if val != nil {
-			t.Errorf("Expected no value to exist")
+			t.Error("Expected no value to exist")
 		}
 		if idx != 0 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "" {
-			t.Errorf("Unexpected value %v", string(val))
+		if string(val) != expected {
+			t.Errorf("Unexpected value %s", val)
+		}
+
+		val = tree.Get(key)
+		if val != nil {
+			t.Error("Fast method - expected no value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %s", val)
 		}
 	}
 
 	// Test "1"
 	{
-		idx, val := tree.Get([]byte("1"))
+		key := []byte("1")
+		expected := "one"
+
+		idx, val := tree.GetWithIndex(key)
 		if val == nil {
-			t.Errorf("Expected value to exist")
+			t.Error("Expected value to exist")
 		}
 		if idx != 0 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "one" {
-			t.Errorf("Unexpected value %v", string(val))
+		if string(val) != expected {
+			t.Errorf("Unexpected value %s", val)
+		}
+
+		val = tree.Get(key)
+		if val == nil {
+			t.Error("Fast method - expected value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %s", val)
 		}
 	}
 
 	// Test "2"
 	{
-		idx, val := tree.Get([]byte("2"))
+		key := []byte("2")
+		expected := "TWO"
+
+		idx, val := tree.GetWithIndex(key)
 		if val == nil {
-			t.Errorf("Expected value to exist")
+			t.Error("Expected value to exist")
 		}
 		if idx != 1 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "TWO" {
-			t.Errorf("Unexpected value %v", string(val))
+		if string(val) != expected {
+			t.Errorf("Unexpected value %s", val)
+		}
+
+		val = tree.Get(key)
+		if val == nil {
+			t.Error("Fast method - expected value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %s", val)
 		}
 	}
 
 	// Test "4"
 	{
-		idx, val := tree.Get([]byte("4"))
+		key := []byte("4")
+		expected := ""
+
+		idx, val := tree.GetWithIndex(key)
 		if val != nil {
-			t.Errorf("Expected no value to exist")
+			t.Error("Expected no value to exist")
 		}
 		if idx != 2 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "" {
-			t.Errorf("Unexpected value %v", string(val))
+		if string(val) != expected {
+			t.Errorf("Unexpected value %s", val)
+		}
+
+		val = tree.Get(key)
+		if val != nil {
+			t.Error("Fast method - expected no value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %s", val)
 		}
 	}
 
 	// Test "6"
 	{
-		idx, val := tree.Get([]byte("6"))
+		key := []byte("6")
+		expected := ""
+
+		idx, val := tree.GetWithIndex(key)
 		if val != nil {
-			t.Errorf("Expected no value to exist")
+			t.Error("Expected no value to exist")
 		}
 		if idx != 3 {
 			t.Errorf("Unexpected idx %x", idx)
 		}
-		if string(val) != "" {
-			t.Errorf("Unexpected value %v", string(val))
+		if string(val) != expected {
+			t.Errorf("Unexpected value %s", val)
+		}
+
+		val = tree.Get(key)
+		if val != nil {
+			t.Error("Fast method - expected no value to exist")
+		}
+		if string(val) != expected {
+			t.Errorf("Fast method - Unexpected value %s", val)
 		}
 	}
 }
@@ -252,7 +307,7 @@ func TestIntegration(t *testing.T) {
 		if has := tree.Has([]byte(randstr(12))); has {
 			t.Error("Table has extra key")
 		}
-		if _, val := tree.Get([]byte(r.key)); string(val) != r.value {
+		if val := tree.Get([]byte(r.key)); string(val) != r.value {
 			t.Error("wrong value")
 		}
 	}
@@ -270,7 +325,7 @@ func TestIntegration(t *testing.T) {
 			if has := tree.Has([]byte(randstr(12))); has {
 				t.Error("Table has extra key")
 			}
-			_, val := tree.Get([]byte(r.key))
+			val := tree.Get([]byte(r.key))
 			if string(val) != r.value {
 				t.Error("wrong value")
 			}
@@ -388,7 +443,7 @@ func TestPersistence(t *testing.T) {
 	require.NoError(t, err)
 	t2.Load()
 	for key, value := range records {
-		_, t2value := t2.Get([]byte(key))
+		t2value := t2.Get([]byte(key))
 		if string(t2value) != value {
 			t.Fatalf("Invalid value. Expected %v, got %v", value, t2value)
 		}

benchmarks/bench_test.go

@@ -19,7 +19,6 @@ func randBytes(length int) []byte {
 	key := make([]byte, length)
 	// math.rand.Read always returns err=nil
 	// we do not need cryptographic randomness for this test:
-	//nolint:gosec
 	rand.Read(key)
 	return key
 }
@@ -57,21 +56,95 @@ func commitTree(b *testing.B, t *iavl.MutableTree) {
 	}
 }
 
-func runQueries(b *testing.B, t *iavl.MutableTree, keyLen int) {
+// queries random keys against live state. Keys are almost certainly not in the tree.
+func runQueriesFast(b *testing.B, t *iavl.MutableTree, keyLen int) {
+	require.True(b, t.IsFastCacheEnabled())
 	for i := 0; i < b.N; i++ {
 		q := randBytes(keyLen)
 		t.Get(q)
 	}
 }
 
-func runKnownQueries(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
+// queries keys that are known to be in state
+func runKnownQueriesFast(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
+	require.True(b, t.IsFastCacheEnabled()) // to ensure fast storage is enabled
 	l := int32(len(keys))
 	for i := 0; i < b.N; i++ {
 		q := keys[rand.Int31n(l)]
 		t.Get(q)
 	}
 }
 
+func runQueriesSlow(b *testing.B, t *iavl.MutableTree, keyLen int) {
+	b.StopTimer()
+	// Save version to get an old immutable tree to query against,
+	// Fast storage is not enabled on old tree versions, allowing us to bench the desired behavior.
+	_, version, err := t.SaveVersion()
+	require.NoError(b, err)
+
+	itree, err := t.GetImmutable(version - 1)
+	require.NoError(b, err)
+	require.False(b, itree.IsFastCacheEnabled()) // to ensure fast storage is not enabled
+
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		q := randBytes(keyLen)
+		itree.GetWithIndex(q)
+	}
+}
+
+func runKnownQueriesSlow(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
+	b.StopTimer()
+	// Save version to get an old immutable tree to query against,
+	// Fast storage is not enabled on old tree versions, allowing us to bench the desired behavior.
+	_, version, err := t.SaveVersion()
+	require.NoError(b, err)
+
+	itree, err := t.GetImmutable(version - 1)
+	require.NoError(b, err)
+	require.False(b, itree.IsFastCacheEnabled()) // to ensure fast storage is not enabled
+	b.StartTimer()
+	l := int32(len(keys))
+	for i := 0; i < b.N; i++ {
+		q := keys[rand.Int31n(l)]
+		index, value := itree.GetWithIndex(q)
+		require.True(b, index >= 0, "the index must not be negative")
+		require.NotNil(b, value, "the value should exist")
+	}
+}
+
+func runIterationFast(b *testing.B, t *iavl.MutableTree, expectedSize int) {
+	require.True(b, t.IsFastCacheEnabled()) // to ensure fast storage is enabled
+	for i := 0; i < b.N; i++ {
+		itr := t.ImmutableTree.Iterator(nil, nil, false)
+		iterate(b, itr, expectedSize)
+		require.Nil(b, itr.Close(), ".Close should not error out")
+	}
+}
+
+func runIterationSlow(b *testing.B, t *iavl.MutableTree, expectedSize int) {
+	for i := 0; i < b.N; i++ {
+		itr := iavl.NewIterator(nil, nil, false, t.ImmutableTree) // create slow iterator directly
+		iterate(b, itr, expectedSize)
+		require.Nil(b, itr.Close(), ".Close should not error out")
+	}
+}
+
+func iterate(b *testing.B, itr db.Iterator, expectedSize int) {
+	b.StartTimer()
+	keyValuePairs := make([][][]byte, 0, expectedSize)
+	for i := 0; i < expectedSize && itr.Valid(); i++ {
+		itr.Next()
+		keyValuePairs = append(keyValuePairs, [][]byte{itr.Key(), itr.Value()})
+	}
+	b.StopTimer()
+	if g, w := len(keyValuePairs), expectedSize; g != w {
+		b.Errorf("iteration count mismatch: got=%d, want=%d", g, w)
+	} else {
+		b.Logf("completed %d iterations", len(keyValuePairs))
+	}
+}
+
 // func runInsert(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int) *iavl.MutableTree {
 // 	for i := 1; i <= b.N; i++ {
 // 		t.Set(randBytes(keyLen), randBytes(dataLen))
@@ -132,7 +205,7 @@ func runBlock(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int,
 			data := randBytes(dataLen)
 
 			// perform query and write on check and then real
-			// check.Get(key)
+			// check.GetFast(key)
 			// check.Set(key, data)
 			real.Get(key)
 			real.Set(key, data)
@@ -175,11 +248,11 @@ func BenchmarkMedium(b *testing.B) {
 	benchmarks := []benchmark{
 		{"memdb", 100000, 100, 16, 40},
 		{"goleveldb", 100000, 100, 16, 40},
-		{"cleveldb", 100000, 100, 16, 40},
+		// {"cleveldb", 100000, 100, 16, 40},
 		// FIXME: idk why boltdb is too slow !?
 		// {"boltdb", 100000, 100, 16, 40},
-		{"rocksdb", 100000, 100, 16, 40},
-		{"badgerdb", 100000, 100, 16, 40},
+		// {"rocksdb", 100000, 100, 16, 40},
+		// {"badgerdb", 100000, 100, 16, 40},
 	}
 	runBenchmarks(b, benchmarks)
 }
@@ -188,10 +261,10 @@ func BenchmarkSmall(b *testing.B) {
 	benchmarks := []benchmark{
 		{"memdb", 1000, 100, 4, 10},
 		{"goleveldb", 1000, 100, 4, 10},
-		{"cleveldb", 1000, 100, 4, 10},
-		{"boltdb", 1000, 100, 4, 10},
-		{"rocksdb", 1000, 100, 4, 10},
-		{"badgerdb", 1000, 100, 4, 10},
+		// {"cleveldb", 1000, 100, 4, 10},
+		// {"boltdb", 1000, 100, 4, 10},
+		// {"rocksdb", 1000, 100, 4, 10},
+		// {"badgerdb", 1000, 100, 4, 10},
 	}
 	runBenchmarks(b, benchmarks)
 }
@@ -202,8 +275,8 @@ func BenchmarkLarge(b *testing.B) {
 		{"goleveldb", 1000000, 100, 16, 40},
 		// FIXME: idk why boltdb is too slow !?
 		// {"boltdb", 1000000, 100, 16, 40},
-		{"rocksdb", 1000000, 100, 16, 40},
-		{"badgerdb", 1000000, 100, 16, 40},
+		// {"rocksdb", 1000000, 100, 16, 40},
+		// {"badgerdb", 1000000, 100, 16, 40},
 	}
 	runBenchmarks(b, benchmarks)
 }
@@ -287,14 +360,38 @@ func runSuite(b *testing.B, d db.DB, initSize, blockSize, keyLen, dataLen int) {
 
 	b.ResetTimer()
 
-	b.Run("query-miss", func(sub *testing.B) {
+	b.Run("query-no-in-tree-guarantee-fast", func(sub *testing.B) {
+		sub.ReportAllocs()
+		runQueriesFast(sub, t, keyLen)
+	})
+	b.Run("query-no-in-tree-guarantee-slow", func(sub *testing.B) {
+		sub.ReportAllocs()
+		runQueriesSlow(sub, t, keyLen)
+	})
+	//
+	b.Run("query-hits-fast", func(sub *testing.B) {
 		sub.ReportAllocs()
-		runQueries(sub, t, keyLen)
+		runKnownQueriesFast(sub, t, keys)
 	})
-	b.Run("query-hits", func(sub *testing.B) {
+	b.Run("query-hits-slow", func(sub *testing.B) {
 		sub.ReportAllocs()
-		runKnownQueries(sub, t, keys)
+		runKnownQueriesSlow(sub, t, keys)
 	})
+	//
+	// Iterations for BenchmarkLevelDBLargeData timeout bencher in CI so
+	// we must skip them.
+	if b.Name() != "BenchmarkLevelDBLargeData" {
+		b.Run("iteration-fast", func(sub *testing.B) {
+			sub.ReportAllocs()
+			runIterationFast(sub, t, initSize)
+		})
+		b.Run("iteration-slow", func(sub *testing.B) {
+			sub.ReportAllocs()
+			runIterationSlow(sub, t, initSize)
+		})
+	}
+
+	//
 	b.Run("update", func(sub *testing.B) {
 		sub.ReportAllocs()
 		t = runUpdate(sub, t, dataLen, blockSize, keys)