Skip to content

Commit

Permalink
manifest: clean up Level
Browse files Browse the repository at this point in the history
Level can now also refer to flushable ingests. This change renames it
to `Layer` and makes it easier and less error-prone to use.
  • Loading branch information
RaduBerinde committed Jul 30, 2024
1 parent 368636f commit cde3df0
Show file tree
Hide file tree
Showing 20 changed files with 173 additions and 118 deletions.
6 changes: 3 additions & 3 deletions compaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -738,7 +738,7 @@ func (c *compaction) newInputIters(
}
}
} else {
addItersForLevel := func(level *compactionLevel, l manifest.Level) error {
addItersForLevel := func(level *compactionLevel, l manifest.Layer) error {
// Add a *levelIter for point iterators. Because we don't call
// initRangeDel, the levelIter will close and forget the range
// deletion iterator when it steps on to a new file. Surfacing range
Expand Down Expand Up @@ -899,9 +899,9 @@ func (c *compaction) newInputIters(
}

func (c *compaction) newRangeDelIter(
newIters tableNewIters, f manifest.LevelFile, opts IterOptions, l manifest.Level,
newIters tableNewIters, f manifest.LevelFile, opts IterOptions, l manifest.Layer,
) (*noCloseIter, error) {
opts.level = l
opts.layer = l
iterSet, err := newIters(context.Background(), f.FileMetadata, &opts,
internalIterOpts{
compaction: true,
Expand Down
2 changes: 1 addition & 1 deletion compaction_picker.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (s sortCompactionLevelsByPriority) Swap(i, j int) {
// sublevel.
type sublevelInfo struct {
manifest.LevelSlice
sublevel manifest.Level
sublevel manifest.Layer
}

func (cl sublevelInfo) Clone() sublevelInfo {
Expand Down
2 changes: 1 addition & 1 deletion db.go
Original file line number Diff line number Diff line change
Expand Up @@ -1475,7 +1475,7 @@ func (i *Iterator) constructPointIter(
mlevels = mlevels[:numMergingLevels]
levels = levels[:numLevelIters]
i.opts.snapshotForHideObsoletePoints = buf.dbi.seqNum
addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) {
addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Layer) {
li := &levels[levelsIndex]

li.init(ctx, i.opts, &i.comparer, i.newIters, files, level, internalOpts)
Expand Down
2 changes: 1 addition & 1 deletion download.go
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ func (d *DB) newDownloadSpanTask(vers *version, sp DownloadSpan) (_ *downloadSpa
// [sp.StartKey, sp.EndKey). Expand the bounds to the left so that we
// include the start keys of any external sstables that overlap with
// sp.StartKey.
vers.IterAllLevelsAndSublevels(func(iter manifest.LevelIterator, level, sublevel int) {
vers.IterAllLevelsAndSublevels(func(iter manifest.LevelIterator, level manifest.Layer) {
if f := iter.SeekGE(d.cmp, sp.StartKey); f != nil &&
objstorage.IsExternalTable(d.objProvider, f.FileBacking.DiskFileNum) &&
d.cmp(f.Smallest.UserKey, bounds.Start) < 0 {
Expand Down
6 changes: 3 additions & 3 deletions flushable.go
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator {
opts = *o
}
return newLevelIter(
context.Background(), opts, s.comparer, s.newIters, s.slice.Iter(), manifest.FlushableIngestLevel(),
context.Background(), opts, s.comparer, s.newIters, s.slice.Iter(), manifest.FlushableIngestsLayer(),
internalIterOpts{},
)
}
Expand Down Expand Up @@ -248,7 +248,7 @@ func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIter
return keyspanimpl.NewLevelIter(
context.TODO(),
keyspan.SpanIterOptions{}, s.comparer.Compare,
s.constructRangeDelIter, s.slice.Iter(), manifest.FlushableIngestLevel(),
s.constructRangeDelIter, s.slice.Iter(), manifest.FlushableIngestsLayer(),
manifest.KeyTypePoint,
)
}
Expand All @@ -262,7 +262,7 @@ func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIter
return keyspanimpl.NewLevelIter(
context.TODO(),
keyspan.SpanIterOptions{}, s.comparer.Compare, s.newRangeKeyIters,
s.slice.Iter(), manifest.FlushableIngestLevel(), manifest.KeyTypeRange,
s.slice.Iter(), manifest.FlushableIngestsLayer(), manifest.KeyTypeRange,
)
}

Expand Down
4 changes: 2 additions & 2 deletions get_iter.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func (g *getIter) initializeNextIterator() (ok bool) {
// split user keys across adjacent sstables within a level, ensuring that at
// most one sstable overlaps g.key.
func (g *getIter) getSSTableIterators(
files manifest.LevelIterator, level manifest.Level,
files manifest.LevelIterator, level manifest.Layer,
) (internalIterator, keyspan.FragmentIterator, error) {
files = files.Filter(manifest.KeyTypePoint)
m := files.SeekGE(g.comparer.Compare, g.key)
Expand All @@ -270,7 +270,7 @@ func (g *getIter) getSSTableIterators(
return emptyIter, nil, nil
}
// m may possibly contain point (or range deletion) keys relevant to g.key.
g.iterOpts.level = level
g.iterOpts.layer = level
iters, err := g.newIters(context.Background(), m, &g.iterOpts, internalIterOpts{}, iterPointKeys|iterRangeDeletions)
if err != nil {
return emptyIter, nil, err
Expand Down
2 changes: 1 addition & 1 deletion ingest.go
Original file line number Diff line number Diff line change
Expand Up @@ -1727,7 +1727,7 @@ func (d *DB) excise(
Category: "pebble-ingest",
QoSLevel: sstable.LatencySensitiveQoSLevel,
},
level: manifest.Level(level),
layer: manifest.Level(level),
}, internalIterOpts{}, iterPointKeys|iterRangeDeletions|iterRangeKeys)
itersLoaded = true
return err
Expand Down
4 changes: 2 additions & 2 deletions ingest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -837,9 +837,9 @@ func TestExcise(t *testing.T) {
d.mu.Unlock()
current := d.mu.versions.currentVersion()

current.IterAllLevelsAndSublevels(func(iter manifest.LevelIterator, level, _ int) {
current.IterAllLevelsAndSublevels(func(iter manifest.LevelIterator, l manifest.Layer) {
for m := iter.SeekGE(d.cmp, exciseSpan.Start); m != nil && d.cmp(m.Smallest.UserKey, exciseSpan.End) < 0; m = iter.Next() {
_, err := d.excise(exciseSpan.UserKeyBounds(), m, ve, level)
_, err := d.excise(exciseSpan.UserKeyBounds(), m, ve, l.Level())
if err != nil {
td.Fatalf(t, "error when excising %s: %s", m.FileNum, err.Error())
}
Expand Down
8 changes: 4 additions & 4 deletions internal/keyspan/keyspanimpl/level_iter.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ type TableNewSpanIter func(
type LevelIter struct {
cmp base.Compare
keyType manifest.KeyType
// The LSM level this LevelIter is initialized for. Used in logging.
level manifest.Level
// The LSM level this LevelIter is initialized for.
level manifest.Layer
// newIter creates a range del iterator if keyType is KeyTypePoint or a range
// key iterator if keyType is KeyTypeRange.
newIter TableNewSpanIter
Expand Down Expand Up @@ -87,7 +87,7 @@ func NewLevelIter(
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
level manifest.Layer,
keyType manifest.KeyType,
) *LevelIter {
l := &LevelIter{}
Expand All @@ -105,7 +105,7 @@ func (l *LevelIter) Init(
cmp base.Compare,
newIter TableNewSpanIter,
files manifest.LevelIterator,
level manifest.Level,
level manifest.Layer,
keyType manifest.KeyType,
) {
if keyType != manifest.KeyTypePoint && keyType != manifest.KeyTypeRange {
Expand Down
4 changes: 2 additions & 2 deletions internal/keyspan/keyspanimpl/level_iter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,7 @@ func TestLevelIterEquivalence(t *testing.T) {
levelIter.Init(
context.Background(),
keyspan.SpanIterOptions{}, base.DefaultComparer.Compare, tableNewIters,
v.Levels[6].Iter(), 0, manifest.KeyTypeRange,
v.Levels[6].Iter(), manifest.Level(0), manifest.KeyTypeRange,
)
levelIters = append(levelIters, &levelIter)
}
Expand Down Expand Up @@ -428,7 +428,7 @@ func TestLevelIter(t *testing.T) {
metas[i] = files[i].meta
}
lm := manifest.MakeLevelMetadata(cmp, 6, metas)
iter := NewLevelIter(context.Background(), keyspan.SpanIterOptions{}, cmp, tableNewIters, lm.Iter(), 6, keyType)
iter := NewLevelIter(context.Background(), keyspan.SpanIterOptions{}, cmp, tableNewIters, lm.Iter(), manifest.Level(6), keyType)
extraInfo := func() string {
return iter.String()
}
Expand Down
121 changes: 121 additions & 0 deletions internal/manifest/layer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package manifest

import (
"fmt"
"math"

"github.com/cockroachdb/redact"
)

// Layer represents a section of the logical sstable hierarchy. It can represent:
// - a level L1 through L6, or
// - the entire L0 level, or
// - a specific L0 sublevel, or
// - the layer of flushable ingests (which is conceptually above the LSM).
type Layer struct {
kind layerKind
value uint16
}

// Level returns a Layer that represents an entire level (L0 through L6).
func Level(level int) Layer {
if level < 0 || level >= NumLevels {
panic("invalid level")
}
return Layer{
kind: levelLayer,
value: uint16(level),
}
}

// L0Sublevel returns a Layer that represents a specific L0 sublevel.
func L0Sublevel(sublevel int) Layer {
// Note: Pebble stops writes once we get to 1000 sublevels.
if sublevel < 0 || sublevel > math.MaxUint16 {
panic("invalid sublevel")
}
return Layer{
kind: l0SublevelLayer,
value: uint16(sublevel),
}
}

// FlushableIngestsLayer returns a Layer that represents the flushable ingests
// layer (which is logically above L0).
func FlushableIngestsLayer() Layer {
return Layer{
kind: flushableIngestsLayer,
}
}

// IsSet returns true if l has been initialized.
func (l Layer) IsSet() bool {
return l.kind != 0
}

// IsFlushableIngests returns true if the layer represents flushable ingests.
func (l Layer) IsFlushableIngests() bool {
return l.kind == flushableIngestsLayer
}

// IsL0Sublevel returns true if the layer represents an L0 sublevel.
func (l Layer) IsL0Sublevel() bool {
return l.kind == l0SublevelLayer
}

// Level returns the level for the layer. Must not be called if
// the layer represents flushable ingests.
func (l Layer) Level() int {
switch l.kind {
case levelLayer:
return int(l.value)
case l0SublevelLayer:
return 0
case flushableIngestsLayer:
panic("flushable ingests layer")
default:
panic("invalid layer")
}
}

// Sublevel returns the L0 sublevel. Can only be called if the layer represents
// an L0 sublevel.
func (l Layer) Sublevel() int {
if !l.IsL0Sublevel() {
panic("not an L0 sublevel layer")
}
return int(l.value)
}

func (l Layer) String() string {
switch l.kind {
case levelLayer:
return fmt.Sprintf("L%d", l.value)
case l0SublevelLayer:
return fmt.Sprintf("L0.%d", l.value)
case flushableIngestsLayer:
return "flushable-ingests"
default:
return "unknown"
}
}

// SafeFormat implements redact.SafeFormatter.
func (l Layer) SafeFormat(s redact.SafePrinter, verb rune) {
s.SafeString(redact.SafeString(l.String()))
}

type layerKind uint8

const (
// Entire level: value contains the level number (0 through 6).
levelLayer layerKind = iota + 1
// L0 sublevel: value contains the sublevel number.
l0SublevelLayer
// Flushable ingests layer: value is unused.
flushableIngestsLayer
)
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ import (
"github.com/stretchr/testify/require"
)

func TestLevel(t *testing.T) {
func TestLayer(t *testing.T) {
testCases := []struct {
level Level
layer Layer
expected string
}{
{Level(0), "L0"},
Expand All @@ -27,12 +27,12 @@ func TestLevel(t *testing.T) {
{L0Sublevel(1), "L0.1"},
{L0Sublevel(2), "L0.2"},

{FlushableIngestLevel(), "flushable-ingest"},
{FlushableIngestsLayer(), "flushable-ingests"},
}

for _, c := range testCases {
t.Run(c.expected, func(t *testing.T) {
s := c.level.String()
s := c.layer.String()
require.EqualValues(t, c.expected, s)
})
}
Expand Down
69 changes: 0 additions & 69 deletions internal/manifest/level.go

This file was deleted.

Loading

0 comments on commit cde3df0

Please sign in to comment.