storage: add experimental MVCC range tombstone primitives

This patch adds initial experimental primitives for MVCC range
tombstones and the range keys they build on, based on experimental
Pebble range keys,

* Data structures:
  * `storage.MVCCRangeKey`
  * `nil` value for range tombstones (as for point tombstones).

* Engine methods for writing, removing, and reading range keys:
  * `Engine.ExperimentalClearMVCCRangeKey()`
  * `Engine.ExperimentalPutMVCCRangeKey()`
  * `SimpleMVCCIterator.HasPointAndRange()`
  * `SimpleMVCCIterator.RangeBounds()`
  * `SimpleMVCCIterator.RangeKeys()`

* MVCC functions and iterator for writing and reading range tombstones:
  * `storage.ExperimentalMVCCDeleteRangeUsingTombstone()`
  * `storage.ScanMVCCTombstones()`
  * `storage.MVCCRangeTombstoneIterator`

Range tombstones do not have a distinct identity, and should instead be
considered a tombstone continuum: they will merge with abutting
tombstones, can be partially cleared, can split or merge along with
ranges, and so on. Bounded scans will truncate them to the scan bounds.

Raw fragmented range keys can be accessed via the `Engine` and
`SimpleMVCCIterator` interfaces, but this is primarily for internal MVCC
usage. In the externally facing MVCC API, range tombstones will be
exposed primarily in defragmented form.

Range tombstones are not yet handled in the rest of the MVCC API, nor
are they exposed via KV APIs. They are not persisted to disk either, due
to Pebble range key limitations. Subsequent pull requests will extend
their functionality and integrate them with other components.

Release note: None

pkg/kv/kvserver/rangefeed/task_test.go

@@ -190,6 +190,21 @@ func (s *testIterator) curKV() storage.MVCCKeyValue {
 	return s.kvs[s.cur]
 }
 
+// HasPointAndRange implements SimpleMVCCIterator.
+func (s *testIterator) HasPointAndRange() (bool, bool) {
+	panic("not implemented")
+}
+
+// RangeBounds implements SimpleMVCCIterator.
+func (s *testIterator) RangeBounds() (roachpb.Key, roachpb.Key) {
+	panic("not implemented")
+}
+
+// RangeTombstones implements SimpleMVCCIterator.
+func (s *testIterator) RangeKeys() []storage.MVCCRangeKeyValue {
+	panic("not implemented")
+}
+
 func TestInitResolvedTSScan(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	startKey := roachpb.RKey("d")

pkg/kv/kvserver/spanset/batch.go

@@ -176,6 +176,21 @@ func (i *MVCCIterator) UnsafeValue() []byte {
 	return i.i.UnsafeValue()
 }
 
+// HasPointAndRange implements SimpleMVCCIterator.
+func (i *MVCCIterator) HasPointAndRange() (bool, bool) {
+	panic("not implemented")
+}
+
+// RangeBounds implements SimpleMVCCIterator.
+func (i *MVCCIterator) RangeBounds() (roachpb.Key, roachpb.Key) {
+	panic("not implemented")
+}
+
+// RangeKeys implements SimpleMVCCIterator.
+func (i *MVCCIterator) RangeKeys() []storage.MVCCRangeKeyValue {
+	panic("not implemented")
+}
+
 // ComputeStats is part of the storage.MVCCIterator interface.
 func (i *MVCCIterator) ComputeStats(
 	start, end roachpb.Key, nowNanos int64,
@@ -599,6 +614,22 @@ func (s spanSetWriter) ClearIterRange(iter storage.MVCCIterator, start, end roac
 	return s.w.ClearIterRange(iter, start, end)
 }
 
+func (s spanSetWriter) ExperimentalPutMVCCRangeKey(
+	rangeKey storage.MVCCRangeKey, value []byte,
+) error {
+	if err := s.checkAllowedRange(rangeKey.StartKey, rangeKey.EndKey); err != nil {
+		return err
+	}
+	return s.w.ExperimentalPutMVCCRangeKey(rangeKey, value)
+}
+
+func (s spanSetWriter) ExperimentalClearMVCCRangeKey(rangeKey storage.MVCCRangeKey) error {
+	if err := s.checkAllowedRange(rangeKey.StartKey, rangeKey.EndKey); err != nil {
+		return err
+	}
+	return s.w.ExperimentalClearMVCCRangeKey(rangeKey)
+}
+
 func (s spanSetWriter) Merge(key storage.MVCCKey, value []byte) error {
 	if s.spansOnly {
 		if err := s.spans.CheckAllowed(SpanReadWrite, roachpb.Span{Key: key.Key}); err != nil {

pkg/storage/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "mvcc_incremental_iterator.go",
         "mvcc_key.go",
         "mvcc_logical_ops.go",
+        "mvcc_range_tombstone_iterator.go",
         "open.go",
         "pebble.go",
         "pebble_batch.go",
@@ -108,6 +109,7 @@ go_test(
         "mvcc_incremental_iterator_test.go",
         "mvcc_key_test.go",
         "mvcc_logical_ops_test.go",
+        "mvcc_range_tombstone_iterator_test.go",
         "mvcc_stats_test.go",
         "mvcc_test.go",
         "pebble_file_registry_test.go",

pkg/storage/engine.go

@@ -72,6 +72,15 @@ type SimpleMVCCIterator interface {
 	// UnsafeValue returns the same value as Value, but the memory is
 	// invalidated on the next call to {Next,NextKey,Prev,SeekGE,SeekLT,Close}.
 	UnsafeValue() []byte
+	// HasPointAndRange returns whether the current iterator position has a point
+	// key and/or a range key. Range keys are only emitted when requested via
+	// IterOptions.KeyTypes.
+	HasPointAndRange() (bool, bool)
+	// RangeBounds returns the range bounds for the current range key, if any.
+	RangeBounds() (roachpb.Key, roachpb.Key)
+	// RangeKeys returns the range key fragments at the current iterator position,
+	// if any.
+	RangeKeys() []MVCCRangeKeyValue
 }
 
 // IteratorStats is returned from {MVCCIterator,EngineIterator}.Stats.
@@ -309,8 +318,26 @@ type IterOptions struct {
 	// use such an iterator is to use it in concert with an iterator without
 	// timestamp hints, as done by MVCCIncrementalIterator.
 	MinTimestampHint, MaxTimestampHint hlc.Timestamp
+	// KeyTypes specifies the types of keys to surface: point and/or range keys.
+	// Use HasPointAndRange() to determine which key type is present at a given
+	// iterator position, and RangeBounds() and RangeKeys() to access range keys.
+	KeyTypes IterKeyType
 }
 
+// IterKeyType configures which types of keys an iterator should surface.
+//
+// TODO(erikgrinaker): Combine this with MVCCIterKind somehow.
+type IterKeyType = pebble.IterKeyType
+
+const (
+	// IterKeyTypePointsOnly iterates over point keys only.
+	IterKeyTypePointsOnly = pebble.IterKeyTypePointsOnly
+	// IterKeyTypePointsAndRanges iterates over both point and range keys.
+	IterKeyTypePointsAndRanges = pebble.IterKeyTypePointsAndRanges
+	// IterKeyTypeRangesOnly iterates over only range keys.
+	IterKeyTypeRangesOnly = pebble.IterKeyTypeRangesOnly
+)
+
 // MVCCIterKind is used to inform Reader about the kind of iteration desired
 // by the caller.
 type MVCCIterKind int
@@ -584,6 +611,38 @@ type Writer interface {
 	// returns.
 	ClearIterRange(iter MVCCIterator, start, end roachpb.Key) error
 
+	// ExperimentalClearMVCCRangeKey deletes an MVCC range key from start
+	// (inclusive) to end (exclusive) at the given timestamp. For any range key
+	// that straddles the start and end boundaries, only the segments within the
+	// boundaries will be cleared. Clears are idempotent.
+	//
+	// This method is primarily intented for MVCC garbage collection and similar
+	// internal use. It mutates MVCC history, and does not check for intents or
+	// other conflicts.
+	//
+	// This method is EXPERIMENTAL. Range keys are not supported throughout the
+	// MVCC API, and the on-disk format is unstable.
+	ExperimentalClearMVCCRangeKey(rangeKey MVCCRangeKey) error
+
+	// ExperimentalPutMVCCRangeKey writes a value to an MVCC range key. It is
+	// currently only used for range tombstones, which have a value of nil.
+	// Range keys exist separately from point keys in Pebble, and must be
+	// accessed via specialized iterator options and methods. See e.g.
+	// IterOptions.KeyTypes and SimpleMVCCIterator.RangeKeys().
+	//
+	// A range key does not have a distinct identity, but should be considered a
+	// key continuum. They will be fragmented by Pebble such that all overlapping
+	// range keys between two fragment bounds form a "stack" of range key
+	// fragments. This fragmentation is non-deterministic, and will also depend on
+	// the internal SST structure (and thus on Pebble compactions) and the store's
+	// write history. They will also split and merge along with CRDB ranges, can
+	// be partially removed via ExperimentalClearMVCCRangeKey, and may be
+	// truncated by bounded scans or iterators.
+	//
+	// This function is EXPERIMENTAL. Range keys are not handled throughout the
+	// MVCC API, and the on-disk format is unstable.
+	ExperimentalPutMVCCRangeKey(MVCCRangeKey, []byte) error
+
 	// Merge is a high-performance write operation used for values which are
 	// accumulated over several writes. Multiple values can be merged
 	// sequentially into a single key; a subsequent read will return a "merged"

pkg/storage/intent_interleaving_iter.go

@@ -715,6 +715,21 @@ func (i *intentInterleavingIter) Value() []byte {
 	return i.iter.Value()
 }
 
+// HasPointAndRange implements SimpleMVCCIterator.
+func (i *intentInterleavingIter) HasPointAndRange() (bool, bool) {
+	panic("not implemented")
+}
+
+// RangeBounds implements SimpleMVCCIterator.
+func (i *intentInterleavingIter) RangeBounds() (roachpb.Key, roachpb.Key) {
+	panic("not implemented")
+}
+
+// RangeKeys implements SimpleMVCCIterator.
+func (i *intentInterleavingIter) RangeKeys() []MVCCRangeKeyValue {
+	panic("not implemented")
+}
+
 func (i *intentInterleavingIter) Close() {
 	i.iter.Close()
 	i.intentIter.Close()

pkg/storage/multi_iterator.go

@@ -14,6 +14,7 @@ import (
 	"bytes"
 
 	"github.com/cockroachdb/cockroach/pkg/keys"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
 )
 
 const invalidIdxSentinel = -1
@@ -92,6 +93,21 @@ func (f *multiIterator) UnsafeValue() []byte {
 	return f.iters[f.currentIdx].UnsafeValue()
 }
 
+// HasPointAndRange implements SimpleMVCCIterator.
+func (f *multiIterator) HasPointAndRange() (bool, bool) {
+	panic("not implemented")
+}
+
+// RangeBounds implements SimpleMVCCIterator.
+func (f *multiIterator) RangeBounds() (roachpb.Key, roachpb.Key) {
+	panic("not implemented")
+}
+
+// RangeKeys implements SimpleMVCCIterator.
+func (f *multiIterator) RangeKeys() []MVCCRangeKeyValue {
+	panic("not implemented")
+}
+
 // Next advances the iterator to the next key/value in the iteration. After this
 // call, Valid() will be true if the iterator was not positioned at the last
 // key.

pkg/storage/mvcc.go

@@ -99,6 +99,12 @@ type MVCCKeyValue struct {
 	Value []byte
 }
 
+// MVCCRangeKeyValue represents a ranged key/value pair.
+type MVCCRangeKeyValue struct {
+	Key   MVCCRangeKey
+	Value []byte
+}
+
 // optionalValue represents an optional roachpb.Value. It is preferred
 // over a *roachpb.Value to avoid the forced heap allocation.
 type optionalValue struct {
@@ -2202,6 +2208,33 @@ func MVCCDeleteRange(
 	return keys, res.ResumeSpan, res.NumKeys, nil
 }
 
+// ExperimentalMVCCDeleteRangeUsingTombstone deletes the given MVCC keyspan at
+// the given timestamp using a range tombstone (rather than point tombstones).
+// This operation is non-transactional, but will check for existing intents and
+// return a WriteIntentError containing up to maxIntents intents.
+//
+// This function is EXPERIMENTAL. Range tombstones are not supported throughout
+// the MVCC API, and the on-disk format is unstable.
+//
+// TODO(erikgrinaker): Needs conflict handling, e.g. WriteTooOldError.
+// TODO(erikgrinaker): Needs MVCCStats handling.
+func ExperimentalMVCCDeleteRangeUsingTombstone(
+	ctx context.Context,
+	rw ReadWriter,
+	ms *enginepb.MVCCStats,
+	startKey, endKey roachpb.Key,
+	timestamp hlc.Timestamp,
+	maxIntents int64,
+) error {
+	if intents, err := ScanIntents(ctx, rw, startKey, endKey, maxIntents, 0); err != nil {
+		return err
+	} else if len(intents) > 0 {
+		return &roachpb.WriteIntentError{Intents: intents}
+	}
+	return rw.ExperimentalPutMVCCRangeKey(MVCCRangeKey{
+		StartKey: startKey, EndKey: endKey, Timestamp: timestamp}, nil)
+}
+
 func recordIteratorStats(traceSpan *tracing.Span, iteratorStats IteratorStats) {
 	stats := iteratorStats.Stats
 	if traceSpan != nil {
@@ -3929,3 +3962,30 @@ func ComputeStatsForRange(
 	ms.LastUpdateNanos = nowNanos
 	return ms, nil
 }
+
+// MVCCScanRangeTombstones returns a list of range tombstones across the given
+// span at the given timestamp, in end,timestamp order rather that
+// start,timestamp. Any tombstones that straddle the bounds will be truncated.
+func MVCCScanRangeTombstones(
+	ctx context.Context, reader Reader, start, end roachpb.Key, ts hlc.Timestamp,
+) ([]MVCCRangeKey, error) {
+	var tombstones []MVCCRangeKey
+	iter := NewMVCCRangeTombstoneIterator(reader, MVCCRangeTombstoneIterOptions{
+		LowerBound:   start,
+		UpperBound:   end,
+		MaxTimestamp: ts,
+	})
+	for {
+		if ok, err := iter.Valid(); err != nil {
+			return nil, err
+		} else if !ok {
+			break
+		}
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		tombstones = append(tombstones, iter.Key())
+		iter.Next()
+	}
+	return tombstones, nil
+}