metamorphic: add testing for synthetic suffix

Add a random synthetic suffix for external ingestion.

One difficulty is that the synthetic suffix has to be larger (as an
integer value) than all suffixes in the object. We can't easily
determine this at generation time, so we generate a random suffix and
at runtime determine if it's ok to use or not.

ingest.go

@@ -1107,35 +1107,42 @@ type ExternalFile struct {
 	// Locator is the shared.Locator that can be used with objProvider to
 	// resolve a reference to this external sstable.
 	Locator remote.Locator
+
 	// ObjName is the unique name of this sstable on Locator.
 	ObjName string
+
 	// Size of the referenced proportion of the virtualized sstable. An estimate
 	// is acceptable in lieu of the backing file size.
 	Size uint64
+
 	// SmallestUserKey and LargestUserKey are the [smallest,largest) user key
 	// bounds of the sstable. Both these bounds are loose i.e. it's possible for
 	// the sstable to not span the entirety of this range. However, multiple
 	// ExternalFiles in one ingestion must all have non-overlapping
 	// [smallest, largest) spans. Note that this Largest bound is exclusive.
 	SmallestUserKey, LargestUserKey []byte
+
 	// HasPointKey and HasRangeKey denote whether this file contains point keys
 	// or range keys. If both structs are false, an error is returned during
 	// ingestion.
 	HasPointKey, HasRangeKey bool
+
 	// ContentPrefix and SyntheticPrefix denote a prefix replacement rule causing
 	// a file, in which all keys have prefix ContentPrefix, to appear whenever it
 	// is accessed as if those keys all instead have prefix SyntheticPrefix.
+	//
 	// SyntheticPrefix must be a prefix of both SmallestUserKey and LargestUserKey.
 	ContentPrefix, SyntheticPrefix []byte
+
 	// SyntheticSuffix will replace the suffix of every key in the file during
 	// iteration. Note that the file itself is not modified, rather, every key
 	// returned by an iterator will have the synthetic suffix.
 	//
-  // The SyntheticSuffix must sort before any non-empty suffixes in the backing
-  // sstable.
-	//
-	// If SyntheticSuffix is set, then SmallestUserKey and LargestUserKey must not
-	// have suffixes.
+	// SyntheticSuffix can only be used under the following conditions:
+	//  - the synthetic suffix must sort before any non-empty suffixes in the
+	//    backing sst;
+	//  - SmallestUserKey and LargestUserKey must not have suffixes;
+	//  - the backing sst must not contain multiple keys with the same prefix.
 	SyntheticSuffix []byte
 }
 

internal/testkeys/testkeys.go

@@ -102,6 +102,11 @@ var Comparer = &base.Comparer{
 	Name:  "pebble.internal.testkeys",
 }
 
+// The comparator is similar to the one in Cockroach; when the prefixes are
+// equal:
+//   - a key without a suffix is smaller than one without a suffix;
+//   - when both keys have a suffix, the key with the larger (decoded) suffix
+//     value is smaller.
 func compare(a, b []byte) int {
 	ai, bi := split(a), split(b)
 	if v := bytes.Compare(a[:ai], b[:bi]); v != 0 {

metamorphic/build.go

@@ -22,19 +22,18 @@ import (
 // writeSSTForIngestion writes an SST that is to be ingested, either directly or
 // as an external file.
 //
-// If convertDelSizedToDel is set, then any DeleteSized keys are converted to
-// Delete keys; this is useful when the database that will ingest the file is at
-// a format that doesn't support DeleteSized.
-//
+// Returns the sstable metadata and the minimum non-empty suffix.
 // Closes the iterators in all cases.
 func writeSSTForIngestion(
 	t *Test,
 	pointIter base.InternalIterator,
 	rangeDelIter keyspan.FragmentIterator,
 	rangeKeyIter keyspan.FragmentIterator,
+	uniquePrefixes bool,
+	syntheticSuffix sstable.SyntheticSuffix,
 	writable objstorage.Writable,
 	targetFMV pebble.FormatMajorVersion,
-) (*sstable.WriterMetadata, error) {
+) (_ *sstable.WriterMetadata, minSuffix []byte, _ error) {
 	writerOpts := t.opts.MakeWriterOptions(0, targetFMV.MaxTableFormat())
 	if t.testOpts.disableValueBlocksForIngestSSTables {
 		writerOpts.DisableValueBlocks = true
@@ -47,11 +46,37 @@ func writeSSTForIngestion(
 	rangeKeyIterCloser := base.CloseHelper(rangeKeyIter)
 	defer rangeKeyIterCloser.Close()
 
+	outputKey := func(key []byte) []byte {
+		n := t.opts.Comparer.Split(key)
+		if suffix := key[n:]; len(suffix) > 0 {
+			if minSuffix == nil || t.opts.Comparer.Compare(suffix, minSuffix) < 0 {
+				minSuffix = slices.Clone(suffix)
+			}
+		}
+		return slices.Clone(key)
+	}
+
+	if len(syntheticSuffix) > 0 {
+		minSuffix = slices.Clone(syntheticSuffix)
+		outputKey = func(key []byte) []byte {
+			n := t.opts.Comparer.Split(key)
+			return append(key[:n:n], syntheticSuffix...)
+		}
+	}
+
 	var lastUserKey []byte
 	for key, value := pointIter.First(); key != nil; key, value = pointIter.Next() {
 		// Ignore duplicate keys.
-		if t.opts.Comparer.Equal(lastUserKey, key.UserKey) {
-			continue
+		if lastUserKey != nil {
+			last := lastUserKey
+			this := key.UserKey
+			if uniquePrefixes {
+				last = last[:t.opts.Comparer.Split(last)]
+				this = this[:t.opts.Comparer.Split(this)]
+			}
+			if t.opts.Comparer.Equal(last, this) {
+				continue
+			}
 		}
 		lastUserKey = append(lastUserKey[:0], key.UserKey...)
 
@@ -65,28 +90,30 @@ func writeSSTForIngestion(
 		}
 		valBytes, _, err := value.Value(nil)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
-		if err := w.Add(*key, valBytes); err != nil {
-			return nil, err
+		k := *key
+		k.UserKey = outputKey(k.UserKey)
+		if err := w.Add(k, valBytes); err != nil {
+			return nil, nil, err
 		}
 	}
 	if err := pointIterCloser.Close(); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if rangeDelIter != nil {
 		span, err := rangeDelIter.First()
 		for ; span != nil; span, err = rangeDelIter.Next() {
-			if err := w.DeleteRange(slices.Clone(span.Start), slices.Clone(span.End)); err != nil {
-				return nil, err
+			if err := w.DeleteRange(outputKey(span.Start), outputKey(span.End)); err != nil {
+				return nil, nil, err
 			}
 		}
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		if err := rangeDelIterCloser.Close(); err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 
@@ -103,8 +130,8 @@ func writeSSTForIngestion(
 			// same sequence number is nonsensical, so we "coalesce" or collapse
 			// the keys.
 			collapsed := keyspan.Span{
-				Start: slices.Clone(span.Start),
-				End:   slices.Clone(span.End),
+				Start: outputKey(span.Start),
+				End:   outputKey(span.End),
 				Keys:  make([]keyspan.Key, 0, len(span.Keys)),
 			}
 			rangekey.Coalesce(
@@ -115,21 +142,25 @@ func writeSSTForIngestion(
 			}
 			keyspan.SortKeysByTrailer(&collapsed.Keys)
 			if err := rangekey.Encode(&collapsed, w.AddRangeKey); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		}
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		if err := rangeKeyIterCloser.Close(); err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 
 	if err := w.Close(); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return w.Metadata()
+	sstMeta, err := w.Metadata()
+	if err != nil {
+		return nil, nil, err
+	}
+	return sstMeta, minSuffix, nil
 }
 
 // buildForIngest builds a local SST file containing the keys in the given batch
@@ -147,9 +178,11 @@ func buildForIngest(
 	iter, rangeDelIter, rangeKeyIter := private.BatchSort(b)
 
 	writable := objstorageprovider.NewFileWritable(f)
-	meta, err := writeSSTForIngestion(
+	meta, _, err := writeSSTForIngestion(
 		t,
 		iter, rangeDelIter, rangeKeyIter,
+		false, /* uniquePrefixes */
+		nil,   /* syntheticSuffix */
 		writable,
 		db.FormatMajorVersion(),
 	)
@@ -160,7 +193,12 @@ func buildForIngest(
 // external object (truncated to the given bounds) and returns its path and
 // metadata.
 func buildForIngestExternalEmulation(
-	t *Test, dbID objID, externalObjID objID, bounds pebble.KeyRange, i int,
+	t *Test,
+	dbID objID,
+	externalObjID objID,
+	bounds pebble.KeyRange,
+	syntheticSuffix sstable.SyntheticSuffix,
+	i int,
 ) (path string, _ *sstable.WriterMetadata) {
 	path = t.opts.FS.PathJoin(t.tmpDir, fmt.Sprintf("ext%d-%d", dbID.slot(), i))
 	f, err := t.opts.FS.Create(path)
@@ -170,9 +208,11 @@ func buildForIngestExternalEmulation(
 	defer reader.Close()
 
 	writable := objstorageprovider.NewFileWritable(f)
-	meta, err := writeSSTForIngestion(
+	meta, _, err := writeSSTForIngestion(
 		t,
 		pointIter, rangeDelIter, rangeKeyIter,
+		false, /* uniquePrefixes */
+		syntheticSuffix,
 		writable,
 		t.minFMV(),
 	)

metamorphic/generator.go

@@ -1337,6 +1337,20 @@ func (g *generator) writerIngestExternalFiles() {
 			o.bounds.Start = keys[2*i]
 		}
 		o.bounds.End = keys[2*i+1]
+
+		if g.rng.Intn(2) == 0 {
+			// Try to set a synthetic suffix. We can only do so if the bounds don't
+			// have suffixes, so try to trim them.
+			start := g.keyGenerator.prefix(o.bounds.Start)
+			end := g.keyGenerator.prefix(o.bounds.End)
+			if g.cmp(start, end) < 0 &&
+				(i == 0 || g.cmp(sorted[i-1].bounds.End, start) <= 0) &&
+				(i == len(sorted)-1 || g.cmp(end, sorted[i+1].bounds.Start) <= 0) {
+				o.bounds.Start = start
+				o.bounds.End = end
+				o.syntheticSuffix = g.keyGenerator.UniformSuffix()
+			}
+		}
 	}
 	// The batches we're ingesting may contain single delete tombstones that
 	// when applied to the writer result in nondeterminism in the deleted key.