db: optimize levelIter for non-matching bloom filter

compaction.go

@@ -1028,8 +1028,8 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r
 
 	iterOpts := IterOptions{logger: c.logger}
 	addItersForLevel := func(iters []internalIterator, level *compactionLevel) ([]internalIterator, error) {
-		iters = append(iters, newLevelIter(iterOpts, c.cmp, newIters, level.files.Iter(),
-			manifest.Level(level.level), &c.bytesIterated))
+		iters = append(iters, newLevelIter(iterOpts, c.cmp, nil /* split */, newIters,
+			level.files.Iter(), manifest.Level(level.level), &c.bytesIterated))
 		// Add the range deletion iterator for each file as an independent level
 		// in mergingIter, as opposed to making a levelIter out of those. This
 		// is safer as levelIter expects all keys coming from underlying

db.go

@@ -793,7 +793,7 @@ func finishInitializingIter(buf *iterAlloc) *Iterator {
 			li = &levelIter{}
 		}
 
-		li.init(dbi.opts, dbi.cmp, dbi.newIters, files, level, nil)
+		li.init(dbi.opts, dbi.cmp, dbi.split, dbi.newIters, files, level, nil)
 		li.initRangeDel(&mlevels[0].rangeDelIter)
 		li.initSmallestLargestUserKey(&mlevels[0].smallestUserKey, &mlevels[0].largestUserKey,
 			&mlevels[0].isLargestUserKeyRangeDelSentinel)

get_iter.go

@@ -145,7 +145,8 @@ func (g *getIter) Next() (*InternalKey, []byte) {
 				files := g.l0[n-1].Iter()
 				g.l0 = g.l0[:n-1]
 				iterOpts := IterOptions{logger: g.logger}
-				g.levelIter.init(iterOpts, g.cmp, g.newIters, files, manifest.L0Sublevel(n), nil)
+				g.levelIter.init(iterOpts, g.cmp, nil /* split */, g.newIters,
+					files, manifest.L0Sublevel(n), nil)
 				g.levelIter.initRangeDel(&g.rangeDelIter)
 				g.iter = &g.levelIter
 				g.iterKey, g.iterValue = g.iter.SeekGE(g.key)
@@ -163,7 +164,7 @@ func (g *getIter) Next() (*InternalKey, []byte) {
 		}
 
 		iterOpts := IterOptions{logger: g.logger}
-		g.levelIter.init(iterOpts, g.cmp, g.newIters,
+		g.levelIter.init(iterOpts, g.cmp, nil /* split */, g.newIters,
 			g.version.Levels[g.level].Iter(), manifest.Level(g.level), nil)
 		g.levelIter.initRangeDel(&g.rangeDelIter)
 		g.level++

ingest.go

@@ -417,7 +417,8 @@ func ingestTargetLevel(
 
 	level := baseLevel
 	for ; level < numLevels; level++ {
-		levelIter := newLevelIter(iterOps, cmp, newIters, v.Levels[level].Iter(), manifest.Level(level), nil)
+		levelIter := newLevelIter(iterOps, cmp, nil /* split */, newIters,
+			v.Levels[level].Iter(), manifest.Level(level), nil)
 		var rangeDelIter internalIterator
 		// Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE sets it up for the target file.
 		levelIter.initRangeDel(&rangeDelIter)

iterator_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/datadriven"
+	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/sstable"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/stretchr/testify/require"
@@ -1003,10 +1004,31 @@ func BenchmarkIteratorSeqSeekPrefixGENotFound(b *testing.B) {
 	const restartInterval = 16
 	const levelCount = 5
 	const keyOffset = 100000
-	readers, levelSlices, _ := buildLevelsForMergingIterSeqSeek(
-		b, blockSize, restartInterval, levelCount, keyOffset, false)
-	readersWithTombstone, levelSlicesWithTombstone, _ := buildLevelsForMergingIterSeqSeek(
-		b, blockSize, restartInterval, 1, keyOffset, true)
+
+	var readers [4][][]*sstable.Reader
+	var levelSlices [4][]manifest.LevelSlice
+	indexFunc := func(bloom bool, withTombstone bool) int {
+		index := 0
+		if bloom {
+			index = 2
+		}
+		if withTombstone {
+			index++
+		}
+		return index
+	}
+	for _, bloom := range []bool{false, true} {
+		for _, withTombstone := range []bool{false, true} {
+			index := indexFunc(bloom, withTombstone)
+			levels := levelCount
+			if withTombstone {
+				levels = 1
+			}
+			readers[index], levelSlices[index], _ = buildLevelsForMergingIterSeqSeek(
+				b, blockSize, restartInterval, levels, keyOffset, withTombstone, bloom)
+
+		}
+	}
 	// We will not be seeking to the keys that were written but instead to
 	// keys before the written keys. This is to validate that the optimization
 	// to use Next still functions when mergingIter checks for the prefix
@@ -1018,53 +1040,52 @@ func BenchmarkIteratorSeqSeekPrefixGENotFound(b *testing.B) {
 		keys = append(keys, []byte(fmt.Sprintf("%08d", i)))
 	}
 	for _, skip := range []int{1, 2, 4} {
-		for _, withTombstone := range []bool{false, true} {
-			b.Run(fmt.Sprintf("skip=%d/with-tombstone=%t", skip, withTombstone),
-				func(b *testing.B) {
-					readers := readers
-					levelSlices := levelSlices
-					if withTombstone {
-						readers = readersWithTombstone
-						levelSlices = levelSlicesWithTombstone
-					}
-					m := buildMergingIter(readers, levelSlices)
-					iter := Iterator{
-						cmp:   DefaultComparer.Compare,
-						equal: DefaultComparer.Equal,
-						split: func(a []byte) int { return len(a) },
-						merge: DefaultMerger.Merge,
-						iter:  m,
-					}
-					pos := 0
-					b.ResetTimer()
-					for i := 0; i < b.N; i++ {
-						// When withTombstone=true, and prior to the
-						// optimization to stop early due to a range
-						// tombstone, the iteration would continue into the
-						// next file, and not be able to use Next at the lower
-						// level in the next SeekPrefixGE call. So we would
-						// incur the cost of iterating over all the deleted
-						// keys for every seek. Note that it is not possible
-						// to do a noop optimization in Iterator for the
-						// prefix case, unlike SeekGE/SeekLT, since we don't
-						// know if the iterators inside mergingIter are all
-						// appropriately positioned -- some may not be due to
-						// bloom filters not matching.
-						valid := iter.SeekPrefixGE(keys[pos])
-						if valid {
-							b.Fatalf("key should not be found")
+		for _, bloom := range []bool{false, true} {
+			for _, withTombstone := range []bool{false, true} {
+				b.Run(fmt.Sprintf("skip=%d/bloom=%t/with-tombstone=%t", skip, bloom, withTombstone),
+					func(b *testing.B) {
+						index := indexFunc(bloom, withTombstone)
+						readers := readers[index]
+						levelSlices := levelSlices[index]
+						m := buildMergingIter(readers, levelSlices)
+						iter := Iterator{
+							cmp:   DefaultComparer.Compare,
+							equal: DefaultComparer.Equal,
+							split: func(a []byte) int { return len(a) },
+							merge: DefaultMerger.Merge,
+							iter:  m,
 						}
-						pos += skip
-						if pos >= keyOffset {
-							pos = 0
+						pos := 0
+						b.ResetTimer()
+						for i := 0; i < b.N; i++ {
+							// When withTombstone=true, and prior to the
+							// optimization to stop early due to a range
+							// tombstone, the iteration would continue into the
+							// next file, and not be able to use Next at the lower
+							// level in the next SeekPrefixGE call. So we would
+							// incur the cost of iterating over all the deleted
+							// keys for every seek. Note that it is not possible
+							// to do a noop optimization in Iterator for the
+							// prefix case, unlike SeekGE/SeekLT, since we don't
+							// know if the iterators inside mergingIter are all
+							// appropriately positioned -- some may not be due to
+							// bloom filters not matching.
+							valid := iter.SeekPrefixGE(keys[pos])
+							if valid {
+								b.Fatalf("key should not be found")
+							}
+							pos += skip
+							if pos >= keyOffset {
+								pos = 0
+							}
 						}
-					}
-					b.StopTimer()
-					iter.Close()
-				})
+						b.StopTimer()
+						iter.Close()
+					})
+			}
 		}
 	}
-	for _, r := range [][][]*sstable.Reader{readers, readersWithTombstone} {
+	for _, r := range readers {
 		for i := range r {
 			for j := range r[i] {
 				r[i][j].Close()
@@ -1081,7 +1102,7 @@ func BenchmarkIteratorSeqSeekGEWithBounds(b *testing.B) {
 	const restartInterval = 16
 	const levelCount = 5
 	readers, levelSlices, keys := buildLevelsForMergingIterSeqSeek(
-		b, blockSize, restartInterval, levelCount, 0 /* keyOffset */, false)
+		b, blockSize, restartInterval, levelCount, 0 /* keyOffset */, false, false)
 	m := buildMergingIter(readers, levelSlices)
 	iter := Iterator{
 		cmp:   DefaultComparer.Compare,
@@ -1118,7 +1139,7 @@ func BenchmarkIteratorSeekGENoop(b *testing.B) {
 	const levelCount = 5
 	const keyOffset = 10000
 	readers, levelSlices, _ := buildLevelsForMergingIterSeqSeek(
-		b, blockSize, restartInterval, levelCount, keyOffset, false)
+		b, blockSize, restartInterval, levelCount, keyOffset, false, false)
 	var keys [][]byte
 	for i := 0; i < keyOffset; i++ {
 		keys = append(keys, []byte(fmt.Sprintf("%08d", i)))

level_checker.go

@@ -636,7 +636,7 @@ func checkLevelsInternal(c *checkConfig) (err error) {
 		manifestIter := current.L0Sublevels.Levels[sublevel].Iter()
 		iterOpts := IterOptions{logger: c.logger}
 		li := &levelIter{}
-		li.init(iterOpts, c.cmp, c.newIters, manifestIter,
+		li.init(iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter,
 			manifest.L0Sublevel(sublevel), nil)
 		li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
 		li.initSmallestLargestUserKey(&mlevelAlloc[0].smallestUserKey, nil, nil)
@@ -650,7 +650,8 @@ func checkLevelsInternal(c *checkConfig) (err error) {
 
 		iterOpts := IterOptions{logger: c.logger}
 		li := &levelIter{}
-		li.init(iterOpts, c.cmp, c.newIters, current.Levels[level].Iter(), manifest.Level(level), nil)
+		li.init(iterOpts, c.cmp, nil /* split */, c.newIters,
+			current.Levels[level].Iter(), manifest.Level(level), nil)
 		li.initRangeDel(&mlevelAlloc[0].rangeDelIter)
 		li.initSmallestLargestUserKey(&mlevelAlloc[0].smallestUserKey, nil, nil)
 		mlevelAlloc[0].iter = li

level_iter.go

@@ -45,6 +45,7 @@ type tableNewIters func(
 type levelIter struct {
 	logger Logger
 	cmp    Compare
+	split  Split
 	// The lower/upper bounds for iteration as specified at creation or the most
 	// recent call to SetBounds.
 	lower []byte
@@ -153,22 +154,26 @@ type levelIter struct {
 // levelIter implements the base.InternalIterator interface.
 var _ base.InternalIterator = (*levelIter)(nil)
 
+// newLevelIter returns a levelIter. It is permissible to pass a nil split
+// parameter if the caller is never going to call SeekPrefixGE.
 func newLevelIter(
 	opts IterOptions,
 	cmp Compare,
+	split Split,
 	newIters tableNewIters,
 	files manifest.LevelIterator,
 	level manifest.Level,
 	bytesIterated *uint64,
 ) *levelIter {
 	l := &levelIter{}
-	l.init(opts, cmp, newIters, files, level, bytesIterated)
+	l.init(opts, cmp, split, newIters, files, level, bytesIterated)
 	return l
 }
 
 func (l *levelIter) init(
 	opts IterOptions,
 	cmp Compare,
+	split Split,
 	newIters tableNewIters,
 	files manifest.LevelIterator,
 	level manifest.Level,
@@ -181,6 +186,7 @@ func (l *levelIter) init(
 	l.upper = opts.UpperBound
 	l.tableOpts.TableFilter = opts.TableFilter
 	l.cmp = cmp
+	l.split = split
 	l.iterFile = nil
 	l.newIters = newIters
 	l.files = files
@@ -435,6 +441,17 @@ func (l *levelIter) SeekPrefixGE(
 		}
 		return l.verify(l.largestBoundary, nil)
 	}
+	// It is possible that we are here because bloom filter matching failed.
+	// In that case it is likely that all keys matching the prefix are wholly
+	// within the current file and cannot be in the subsequent file. In that
+	// case we don't want to go to the next file, since loading and seeking in
+	// there has some cost. Additionally, for sparse key spaces, loading the
+	// next file will defeat the optimization for the next SeekPrefixGE that
+	// is called with trySeekUsingNext=true, since for sparse key spaces it is
+	// likely that the next key will also be contained in the current file.
+	if n := l.split(l.iterFile.Largest.UserKey); l.cmp(prefix, l.iterFile.Largest.UserKey[:n]) < 0 {
+		return nil, nil
+	}
 	return l.verify(l.skipEmptyFileForward())
 }
 

level_iter_test.go

@@ -81,7 +81,8 @@ func TestLevelIter(t *testing.T) {
 			}
 
 			iter := newLevelIter(opts, DefaultComparer.Compare,
-				newIters, files.Iter(), manifest.Level(level), nil)
+				func(a []byte) int { return len(a) }, newIters, files.Iter(), manifest.Level(level),
+				nil)
 			defer iter.Close()
 			// Fake up the range deletion initialization.
 			iter.initRangeDel(new(internalIterator))
@@ -122,7 +123,8 @@ func TestLevelIter(t *testing.T) {
 			}
 
 			iter := newLevelIter(opts, DefaultComparer.Compare,
-				newIters2, files.Iter(), manifest.Level(level), nil)
+				func(a []byte) int { return len(a) }, newIters2, files.Iter(),
+				manifest.Level(level), nil)
 			iter.SeekGE([]byte(key))
 			lower, upper := tableOpts.GetLowerBound(), tableOpts.GetUpperBound()
 			return fmt.Sprintf("[%s,%s]\n", lower, upper)
@@ -283,8 +285,9 @@ func TestLevelIterBoundaries(t *testing.T) {
 			}
 			if iter == nil {
 				slice := manifest.NewLevelSliceKeySorted(lt.cmp.Compare, lt.metas)
-				iter = newLevelIter(IterOptions{}, DefaultComparer.Compare, lt.newIters,
-					slice.Iter(), manifest.Level(level), nil)
+				iter = newLevelIter(IterOptions{}, DefaultComparer.Compare,
+					func(a []byte) int { return len(a) }, lt.newIters, slice.Iter(),
+					manifest.Level(level), nil)
 				// Fake up the range deletion initialization.
 				iter.initRangeDel(new(internalIterator))
 			}
@@ -377,8 +380,9 @@ func TestLevelIterSeek(t *testing.T) {
 		case "iter":
 			slice := manifest.NewLevelSliceKeySorted(lt.cmp.Compare, lt.metas)
 			iter := &levelIterTestIter{
-				levelIter: newLevelIter(IterOptions{}, DefaultComparer.Compare, lt.newIters,
-					slice.Iter(), manifest.Level(level), nil),
+				levelIter: newLevelIter(IterOptions{}, DefaultComparer.Compare,
+					func(a []byte) int { return len(a) }, lt.newIters, slice.Iter(),
+					manifest.Level(level), nil),
 			}
 			defer iter.Close()
 			iter.initRangeDel(&iter.rangeDelIter)
@@ -479,8 +483,7 @@ func BenchmarkLevelIterSeekGE(b *testing.B) {
 								iter, err := readers[file.FileNum].NewIter(nil /* lower */, nil /* upper */)
 								return iter, nil, err
 							}
-							l := newLevelIter(IterOptions{}, DefaultComparer.Compare,
-								newIters, metas.Iter(), manifest.Level(level), nil)
+							l := newLevelIter(IterOptions{}, DefaultComparer.Compare, nil, newIters, metas.Iter(), manifest.Level(level), nil)
 							rng := rand.New(rand.NewSource(uint64(time.Now().UnixNano())))
 
 							b.ResetTimer()
@@ -522,8 +525,7 @@ func BenchmarkLevelIterSeqSeekGEWithBounds(b *testing.B) {
 									opts.LowerBound, opts.UpperBound)
 								return iter, nil, err
 							}
-							l := newLevelIter(IterOptions{}, DefaultComparer.Compare,
-								newIters, metas.Iter(), manifest.Level(level), nil)
+							l := newLevelIter(IterOptions{}, DefaultComparer.Compare, nil, newIters, metas.Iter(), manifest.Level(level), nil)
 							// Fake up the range deletion initialization, to resemble the usage
 							// in a mergingIter.
 							l.initRangeDel(new(internalIterator))
@@ -571,7 +573,8 @@ func BenchmarkLevelIterSeqSeekPrefixGE(b *testing.B) {
 			b.Run(fmt.Sprintf("skip=%d/use-next=%t", skip, useNext),
 				func(b *testing.B) {
 					l := newLevelIter(IterOptions{}, DefaultComparer.Compare,
-						newIters, metas.Iter(), manifest.Level(level), nil)
+						func(a []byte) int { return len(a) }, newIters, metas.Iter(),
+						manifest.Level(level), nil)
 					// Fake up the range deletion initialization, to resemble the usage
 					// in a mergingIter.
 					l.initRangeDel(new(internalIterator))
@@ -613,8 +616,7 @@ func BenchmarkLevelIterNext(b *testing.B) {
 								iter, err := readers[file.FileNum].NewIter(nil /* lower */, nil /* upper */)
 								return iter, nil, err
 							}
-							l := newLevelIter(IterOptions{}, DefaultComparer.Compare,
-								newIters, metas.Iter(), manifest.Level(level), nil)
+							l := newLevelIter(IterOptions{}, DefaultComparer.Compare, nil, newIters, metas.Iter(), manifest.Level(level), nil)
 
 							b.ResetTimer()
 							for i := 0; i < b.N; i++ {
@@ -648,8 +650,7 @@ func BenchmarkLevelIterPrev(b *testing.B) {
 								iter, err := readers[file.FileNum].NewIter(nil /* lower */, nil /* upper */)
 								return iter, nil, err
 							}
-							l := newLevelIter(IterOptions{}, DefaultComparer.Compare,
-								newIters, metas.Iter(), manifest.Level(level), nil)
+							l := newLevelIter(IterOptions{}, DefaultComparer.Compare, nil, newIters, metas.Iter(), manifest.Level(level), nil)
 
 							b.ResetTimer()
 							for i := 0; i < b.N; i++ {