Skip to content

Commit

Permalink
storage: introduce test-build assertion iterator
Browse files Browse the repository at this point in the history
Backport cockroachdb#90859, cockroachdb#96685 and cockroachdb#97222 to mangle the buffers returned by Pebble
iterators to ensure MVCC and Cockroach code respects the Pebble iterator memory
lifetimes.

The RangeBounds() and RangeKeys() validity assertions are omitted, since 22.2
allowed these methods to be invoked on an invalid iterator or an iterator
positioned at a point key.

Release note: None
Release justification: Non-production code changes
  • Loading branch information
jbowens committed Feb 24, 2023
1 parent 587350b commit d7bcb56
Show file tree
Hide file tree
Showing 18 changed files with 498 additions and 41 deletions.
2 changes: 2 additions & 0 deletions build/bazelutil/check.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ pkg/cmd/roachtest/BUILD.bazel
EXISTING_CRDB_TEST_BUILD_CONSTRAINTS="
pkg/util/buildutil/crdb_test_off.go://go:build !crdb_test || crdb_test_off
pkg/util/buildutil/crdb_test_on.go://go:build crdb_test && !crdb_test_off
pkg/storage/pebbleiter/crdb_test_off.go://go:build !crdb_test || crdb_test_off
pkg/storage/pebbleiter/crdb_test_on.go://go:build crdb_test && !crdb_test_off
"

if [ -z "${COCKROACH_BAZEL_CHECK_FAST:-}" ]; then
Expand Down
4 changes: 4 additions & 0 deletions pkg/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,7 @@ ALL_TESTS = [
"//pkg/storage/enginepb:enginepb_test",
"//pkg/storage/fs:fs_test",
"//pkg/storage/metamorphic:metamorphic_test",
"//pkg/storage/pebbleiter:pebbleiter_test",
"//pkg/storage:storage_test",
"//pkg/testutils/docker:docker_test",
"//pkg/testutils/floatcmp:floatcmp_test",
Expand Down Expand Up @@ -1805,6 +1806,8 @@ GO_TARGETS = [
"//pkg/storage/fs:fs_test",
"//pkg/storage/metamorphic:metamorphic",
"//pkg/storage/metamorphic:metamorphic_test",
"//pkg/storage/pebbleiter:pebbleiter",
"//pkg/storage/pebbleiter:pebbleiter_test",
"//pkg/storage:storage",
"//pkg/storage:storage_test",
"//pkg/streaming:streaming",
Expand Down Expand Up @@ -2861,6 +2864,7 @@ GET_X_DATA_TARGETS = [
"//pkg/storage/enginepb:get_x_data",
"//pkg/storage/fs:get_x_data",
"//pkg/storage/metamorphic:get_x_data",
"//pkg/storage/pebbleiter:get_x_data",
"//pkg/streaming:get_x_data",
"//pkg/testutils:get_x_data",
"//pkg/testutils/buildutil:get_x_data",
Expand Down
1 change: 1 addition & 0 deletions pkg/kv/kvserver/spanset/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ go_library(
"//pkg/keys",
"//pkg/roachpb",
"//pkg/storage",
"//pkg/storage/pebbleiter",
"//pkg/util/hlc",
"//pkg/util/log",
"//pkg/util/protoutil",
Expand Down
3 changes: 2 additions & 1 deletion pkg/kv/kvserver/spanset/batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/keys"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/storage"
"github.com/cockroachdb/cockroach/pkg/storage/pebbleiter"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/protoutil"
Expand Down Expand Up @@ -392,7 +393,7 @@ func (i *EngineIterator) UnsafeRawEngineKey() []byte {
}

// GetRawIter is part of the storage.EngineIterator interface.
func (i *EngineIterator) GetRawIter() *pebble.Iterator {
func (i *EngineIterator) GetRawIter() pebbleiter.Iterator {
return i.i.GetRawIter()
}

Expand Down
1 change: 1 addition & 0 deletions pkg/storage/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ go_library(
"//pkg/settings/cluster",
"//pkg/storage/enginepb",
"//pkg/storage/fs",
"//pkg/storage/pebbleiter",
"//pkg/util",
"//pkg/util/admission",
"//pkg/util/bufalloc",
Expand Down
7 changes: 4 additions & 3 deletions pkg/storage/disk_map.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

"github.com/cockroachdb/cockroach/pkg/kv/kvserver/diskmap"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/storage/pebbleiter"
"github.com/cockroachdb/cockroach/pkg/util/encoding"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/errors"
Expand Down Expand Up @@ -52,7 +53,7 @@ type pebbleMapBatchWriter struct {
// pebbleMapIterator iterates over the keys of a pebbleMap in sorted order.
type pebbleMapIterator struct {
allowDuplicates bool
iter *pebble.Iterator
iter pebbleiter.Iterator
// makeKey is a function that transforms a key into a byte slice with a prefix
// used to SeekGE() the underlying iterator.
makeKey func(k []byte) []byte
Expand Down Expand Up @@ -114,9 +115,9 @@ func (r *pebbleMap) makeKeyWithSequence(k []byte) []byte {
func (r *pebbleMap) NewIterator() diskmap.SortedDiskMapIterator {
return &pebbleMapIterator{
allowDuplicates: r.allowDuplicates,
iter: r.store.NewIter(&pebble.IterOptions{
iter: pebbleiter.MaybeWrap(r.store.NewIter(&pebble.IterOptions{
UpperBound: roachpb.Key(r.prefix).PrefixEnd(),
}),
})),
makeKey: r.makeKey,
prefix: r.prefix,
}
Expand Down
13 changes: 2 additions & 11 deletions pkg/storage/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
"github.com/cockroachdb/cockroach/pkg/storage/fs"
"github.com/cockroachdb/cockroach/pkg/storage/pebbleiter"
"github.com/cockroachdb/cockroach/pkg/util/envutil"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/iterutil"
Expand Down Expand Up @@ -340,7 +341,7 @@ type EngineIterator interface {
Value() []byte
// GetRawIter is a low-level method only for use in the storage package,
// that returns the underlying pebble Iterator.
GetRawIter() *pebble.Iterator
GetRawIter() pebbleiter.Iterator
// SeekEngineKeyGEWithLimit is similar to SeekEngineKeyGE, but takes an
// additional exclusive upper limit parameter. The limit is semantically
// best-effort, and is an optimization to avoid O(n^2) iteration behavior in
Expand Down Expand Up @@ -1551,17 +1552,7 @@ func assertSimpleMVCCIteratorInvariants(iter SimpleMVCCIterator) error {
rangeKey, value.Value.RawBytes)
}
}

} else {
// Bounds and range keys must be empty.
if bounds := iter.RangeBounds(); !bounds.Equal(roachpb.Span{}) {
return errors.AssertionFailedf("hasRange=false but RangeBounds=%s", bounds)
}
if r := iter.RangeKeys(); !r.IsEmpty() || !r.Bounds.Equal(roachpb.Span{}) {
return errors.AssertionFailedf("hasRange=false but RangeKeys=%s", r)
}
}

return nil
}

Expand Down
4 changes: 0 additions & 4 deletions pkg/storage/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2171,10 +2171,6 @@ func TestEngineRangeKeysUnsupported(t *testing.T) {
hasPoint, hasRange := iter.HasPointAndRange()
require.True(t, hasPoint)
require.False(t, hasRange)
rangeBounds, err := iter.EngineRangeBounds()
require.NoError(t, err)
require.Empty(t, rangeBounds)
require.Empty(t, iter.EngineRangeKeys())

// Exhaust the iterator.
ok, err = iter.NextEngineKey()
Expand Down
10 changes: 5 additions & 5 deletions pkg/storage/mvcc_history_metamorphic_iterator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ func (m *metamorphicIterator) moveAround() {
}

hasPoint, _ := m.it.HasPointAndRange()
rangeKeys := m.it.RangeKeys().Clone()
rangeKeys := rangeKeysIfExist(m.it).Clone()
var rangeKeysIgnoringTime storage.MVCCRangeKeyStack
if iit != nil {
rangeKeysIgnoringTime = iit.RangeKeysIgnoringTime()
Expand Down Expand Up @@ -224,7 +224,7 @@ func (m *metamorphicIterator) moveAround() {
if m.it.UnsafeKey().Equal(cur) {
break // made it
}
printfln("step: %s %s [changed=%t]", m.it.UnsafeKey(), m.it.RangeKeys(), m.it.RangeKeyChanged())
printfln("step: %s %s [changed=%t]", m.it.UnsafeKey(), rangeKeysIfExist(m.it), m.it.RangeKeyChanged())
if iit != nil {
// If we're an incremental iterator with time bounds, and `cur` is not within bounds,
// would miss it if we used Next. So call NextIgnoringTime unconditionally.
Expand All @@ -247,7 +247,7 @@ func (m *metamorphicIterator) moveAround() {
valid, err := m.it.Valid()
require.Nil(m.t, err)
require.True(m.t, valid, "unable to recover original position following SeekLT")
printfln("rev-step: %s %s [changed=%t]", m.it.UnsafeKey(), m.it.RangeKeys(), m.it.RangeKeyChanged())
printfln("rev-step: %s %s [changed=%t]", m.it.UnsafeKey(), rangeKeysIfExist(m.it), m.it.RangeKeyChanged())
if m.it.UnsafeKey().Equal(cur) {
printfln("done")
break // made it
Expand All @@ -268,13 +268,13 @@ func (m *metamorphicIterator) moveAround() {
rangeKeysIgnoringTime2 = iit.RangeKeysIgnoringTime()
}
printfln("recovered position: %s hasPoint=%t, rangeKeys=%s, rangeKeysIgnoringTime=%s",
m.it.UnsafeKey(), hasPoint2, m.it.RangeKeys(), rangeKeysIgnoringTime2)
m.it.UnsafeKey(), hasPoint2, rangeKeysIfExist(m.it), rangeKeysIgnoringTime2)
}
// Back where we started and hopefully in an indistinguishable state.
// When the stack is empty, sometimes it's a nil slice and sometimes zero
// slice. A similar problem exists with MVCCRangeKeyVersion.Value. Sidestep
// them by comparing strings.
require.Equal(m.t, fmt.Sprint(rangeKeys), fmt.Sprint(m.it.RangeKeys()))
require.Equal(m.t, fmt.Sprint(rangeKeys), fmt.Sprint(rangeKeysIfExist(m.it)))
if iit != nil {
require.Equal(m.t, fmt.Sprint(rangeKeysIgnoringTime), fmt.Sprint(iit.RangeKeysIgnoringTime()))
}
Expand Down
11 changes: 10 additions & 1 deletion pkg/storage/mvcc_history_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1868,9 +1868,18 @@ func printIter(e *evalCtx) {
}
}

func rangeKeysIfExist(it storage.SimpleMVCCIterator) storage.MVCCRangeKeyStack {
if valid, err := it.Valid(); !valid || err != nil {
return storage.MVCCRangeKeyStack{}
} else if _, hasRange := it.HasPointAndRange(); !hasRange {
return storage.MVCCRangeKeyStack{}
}
return it.RangeKeys()
}

func checkAndUpdateRangeKeyChanged(e *evalCtx) bool {
rangeKeyChanged := e.iter.RangeKeyChanged()
rangeKeys := e.iter.RangeKeys()
rangeKeys := rangeKeysIfExist(e.iter)

if incrIter := e.tryMVCCIncrementalIter(); incrIter != nil {
// For MVCCIncrementalIterator, make sure RangeKeyChangedIgnoringTime() fires
Expand Down
5 changes: 3 additions & 2 deletions pkg/storage/pebble.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
"github.com/cockroachdb/cockroach/pkg/storage/fs"
"github.com/cockroachdb/cockroach/pkg/storage/pebbleiter"
"github.com/cockroachdb/cockroach/pkg/util/buildutil"
"github.com/cockroachdb/cockroach/pkg/util/envutil"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
Expand Down Expand Up @@ -1912,7 +1913,7 @@ type pebbleReadOnly struct {
prefixEngineIter pebbleIterator
normalEngineIter pebbleIterator

iter *pebble.Iterator
iter pebbleiter.Iterator
iterUsed bool // avoids cloning after PinEngineStateForIterators()
durability DurabilityRequirement
closed bool
Expand Down Expand Up @@ -2086,7 +2087,7 @@ func (p *pebbleReadOnly) PinEngineStateForIterators() error {
if p.durability == GuaranteedDurability {
o = &pebble.IterOptions{OnlyReadGuaranteedDurable: true}
}
p.iter = p.parent.db.NewIter(o)
p.iter = pebbleiter.MaybeWrap(p.parent.db.NewIter(o))
// NB: p.iterUsed == false avoids cloning this in NewMVCCIterator(), since
// we've just created it.
}
Expand Down
7 changes: 4 additions & 3 deletions pkg/storage/pebble_batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/storage/pebbleiter"
"github.com/cockroachdb/cockroach/pkg/util/uuid"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble"
Expand Down Expand Up @@ -45,7 +46,7 @@ type pebbleBatch struct {
prefixEngineIter pebbleIterator
normalEngineIter pebbleIterator

iter *pebble.Iterator
iter pebbleiter.Iterator
iterUsed bool // avoids cloning after PinEngineStateForIterators()
writeOnly bool
closed bool
Expand Down Expand Up @@ -243,9 +244,9 @@ func (p *pebbleBatch) SupportsRangeKeys() bool {
func (p *pebbleBatch) PinEngineStateForIterators() error {
if p.iter == nil {
if p.batch.Indexed() {
p.iter = p.batch.NewIter(nil)
p.iter = pebbleiter.MaybeWrap(p.batch.NewIter(nil))
} else {
p.iter = p.db.NewIter(nil)
p.iter = pebbleiter.MaybeWrap(p.db.NewIter(nil))
}
// NB: p.iterUsed == false avoids cloning this in NewMVCCIterator(). We've
// just created it, so cloning it would just be overhead.
Expand Down
30 changes: 19 additions & 11 deletions pkg/storage/pebble_iterator.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

"github.com/cockroachdb/cockroach/pkg/keys"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/storage/pebbleiter"
"github.com/cockroachdb/cockroach/pkg/util"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/protoutil"
Expand All @@ -31,7 +32,7 @@ import (
// should only be used in one of the two modes.
type pebbleIterator struct {
// Underlying iterator for the DB.
iter *pebble.Iterator
iter pebbleiter.Iterator
options pebble.IterOptions
// Reusable buffer for MVCCKey or EngineKey encoding.
keyBuf []byte
Expand Down Expand Up @@ -91,27 +92,30 @@ func newPebbleIterator(
p := pebbleIterPool.Get().(*pebbleIterator)
p.reusable = false // defensive
p.init(nil, opts, durability, supportsRangeKeys)
p.iter = handle.NewIter(&p.options)
p.iter = pebbleiter.MaybeWrap(handle.NewIter(&p.options))
return p
}

// newPebbleIteratorByCloning creates a new Pebble iterator by cloning the given
// iterator and reconfiguring it.
func newPebbleIteratorByCloning(
iter *pebble.Iterator, opts IterOptions, durability DurabilityRequirement, supportsRangeKeys bool,
iter pebbleiter.Iterator,
opts IterOptions,
durability DurabilityRequirement,
supportsRangeKeys bool,
) *pebbleIterator {
var err error
p := pebbleIterPool.Get().(*pebbleIterator)
p.reusable = false // defensive
p.init(nil, opts, durability, supportsRangeKeys)
p.iter, err = iter.Clone(pebble.CloneOptions{
iter, err := iter.Clone(pebble.CloneOptions{
IterOptions: &p.options,
RefreshBatchView: true,
})
if err != nil {
p.Close()
panic(err)
}
p.iter = iter
return p
}

Expand All @@ -128,11 +132,12 @@ func newPebbleSSTIterator(
externalIterOpts = append(externalIterOpts, pebble.ExternalIterForwardOnly{})
}

var err error
if p.iter, err = pebble.NewExternalIter(DefaultPebbleOptions(), &p.options, files, externalIterOpts...); err != nil {
iter, err := pebble.NewExternalIter(DefaultPebbleOptions(), &p.options, files, externalIterOpts...)
if err != nil {
p.Close()
return nil, err
}
p.iter = pebbleiter.MaybeWrap(iter)
p.external = true
return p, nil
}
Expand All @@ -141,7 +146,10 @@ func newPebbleSSTIterator(
// reconfiguring the given iter. It is valid to pass a nil iter and then create
// p.iter using p.options, to avoid redundant reconfiguration via SetOptions().
func (p *pebbleIterator) init(
iter *pebble.Iterator, opts IterOptions, durability DurabilityRequirement, supportsRangeKeys bool,
iter pebbleiter.Iterator,
opts IterOptions,
durability DurabilityRequirement,
supportsRangeKeys bool,
) {
*p = pebbleIterator{
iter: iter,
Expand All @@ -164,7 +172,7 @@ func (p *pebbleIterator) init(
// 3. iter == nil: create a new iterator from handle.
func (p *pebbleIterator) initReuseOrCreate(
handle pebble.Reader,
iter *pebble.Iterator,
iter pebbleiter.Iterator,
clone bool,
opts IterOptions,
durability DurabilityRequirement,
Expand All @@ -177,7 +185,7 @@ func (p *pebbleIterator) initReuseOrCreate(

p.init(nil, opts, durability, supportsRangeKeys)
if iter == nil {
p.iter = handle.NewIter(&p.options)
p.iter = pebbleiter.MaybeWrap(handle.NewIter(&p.options))
} else if clone {
var err error
p.iter, err = iter.Clone(pebble.CloneOptions{
Expand Down Expand Up @@ -917,7 +925,7 @@ func (p *pebbleIterator) IsPrefix() bool {
}

// GetRawIter is part of the EngineIterator interface.
func (p *pebbleIterator) GetRawIter() *pebble.Iterator {
func (p *pebbleIterator) GetRawIter() pebbleiter.Iterator {
return p.iter
}

Expand Down
Loading

0 comments on commit d7bcb56

Please sign in to comment.