define FormatSyntheticPrefixes version

format_major_version.go

@@ -178,6 +178,13 @@ const (
 	// a format major version.
 	FormatVirtualSSTables
 
+	// FormatSyntheticPrefixes is a format major version that adds support for
+	// sstables to have their content exposed in a different prefix of keyspace
+	// than the actual prefix persisted in the keys in such sstables. The prefix
+	// replacement information is stored in new fields in the Manifest and thus
+	// requires a format major version.
+	FormatSyntheticPrefixes
+
 	// -- Add new versions here --
 
 	// FormatNewest is the most recent format major version.
@@ -209,7 +216,7 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 	switch v {
 	case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
 		return sstable.TableFormatPebblev3
-	case FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
+	case FormatDeleteSizedAndObsolete, FormatVirtualSSTables, FormatSyntheticPrefixes:
 		return sstable.TableFormatPebblev4
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -221,7 +228,7 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
 	switch v {
 	case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
-		FormatDeleteSizedAndObsolete, FormatVirtualSSTables:
+		FormatDeleteSizedAndObsolete, FormatVirtualSSTables, FormatSyntheticPrefixes:
 		return sstable.TableFormatPebblev1
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -255,6 +262,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 	FormatVirtualSSTables: func(d *DB) error {
 		return d.finalizeFormatVersUpgrade(FormatVirtualSSTables)
 	},
+	FormatSyntheticPrefixes: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatSyntheticPrefixes)
+	},
 }
 
 const formatVersionMarkerName = `format-version`

format_major_version_test.go

@@ -23,11 +23,12 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
 	require.Equal(t, FormatPrePebblev1MarkedCompacted, FormatMajorVersion(14))
 	require.Equal(t, FormatDeleteSizedAndObsolete, FormatMajorVersion(15))
 	require.Equal(t, FormatVirtualSSTables, FormatMajorVersion(16))
+	require.Equal(t, FormatSyntheticPrefixes, FormatMajorVersion(17))
 
 	// When we add a new version, we should add a check for the new version in
 	// addition to updating these expected values.
-	require.Equal(t, FormatNewest, FormatMajorVersion(16))
-	require.Equal(t, internalFormatNewest, FormatMajorVersion(16))
+	require.Equal(t, FormatNewest, FormatMajorVersion(17))
+	require.Equal(t, internalFormatNewest, FormatMajorVersion(17))
 }
 
 func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
@@ -50,6 +51,8 @@ func TestRatchetFormat(t *testing.T) {
 	require.Equal(t, FormatDeleteSizedAndObsolete, d.FormatMajorVersion())
 	require.NoError(t, d.RatchetFormatMajorVersion(FormatVirtualSSTables))
 	require.Equal(t, FormatVirtualSSTables, d.FormatMajorVersion())
+	require.NoError(t, d.RatchetFormatMajorVersion(FormatSyntheticPrefixes))
+	require.Equal(t, FormatSyntheticPrefixes, d.FormatMajorVersion())
 
 	require.NoError(t, d.Close())
 
@@ -203,6 +206,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
 		FormatPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
 		FormatDeleteSizedAndObsolete:     {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
 		FormatVirtualSSTables:            {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
+		FormatSyntheticPrefixes:          {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
 	}
 
 	// Valid versions.

ingest.go

@@ -1308,6 +1308,13 @@ func (d *DB) ingest(
 	if (exciseSpan.Valid() || len(shared) > 0 || len(external) > 0) && d.FormatMajorVersion() < FormatVirtualSSTables {
 		return IngestOperationStats{}, errors.New("pebble: format major version too old for excise, shared or external sstable ingestion")
 	}
+	if len(external) > 0 && d.FormatMajorVersion() < FormatSyntheticPrefixes {
+		for i := range external {
+			if len(external[i].SyntheticPrefix) > 0 {
+				return IngestOperationStats{}, errors.New("pebble: format major version too old for synthetic prefix ingestion")
+			}
+		}
+	}
 	// Allocate file numbers for all of the files being ingested and mark them as
 	// pending in order to prevent them from being deleted. Note that this causes
 	// the file number ordering to be out of alignment with sequence number

ingest_test.go

@@ -1718,7 +1718,7 @@ func TestIngestExternal(t *testing.T) {
 			EventListener: &EventListener{FlushEnd: func(info FlushInfo) {
 				flushed = true
 			}},
-			FormatMajorVersion: FormatVirtualSSTables,
+			FormatMajorVersion: FormatNewest,
 		}
 		opts.Experimental.RemoteStorage = remote.MakeSimpleFactory(map[remote.Locator]remote.Storage{
 			"external-locator": remoteStorage,

open_test.go

@@ -193,7 +193,7 @@ func TestNewDBFilenames(t *testing.T) {
 			"LOCK",
 			"MANIFEST-000001",
 			"OPTIONS-000003",
-			"marker.format-version.000003.016",
+			"marker.format-version.000004.017",
 			"marker.manifest.000001.MANIFEST-000001",
 		},
 	}

testdata/checkpoint

@@ -25,6 +25,10 @@ create: db/marker.format-version.000003.016
 close: db/marker.format-version.000003.016
 remove: db/marker.format-version.000002.015
 sync: db
+create: db/marker.format-version.000004.017
+close: db/marker.format-version.000004.017
+remove: db/marker.format-version.000003.016
+sync: db
 create: db/temporary.000003.dbtmp
 sync: db/temporary.000003.dbtmp
 close: db/temporary.000003.dbtmp
@@ -87,9 +91,9 @@ close:
 open-dir: checkpoints/checkpoint1
 link: db/OPTIONS-000003 -> checkpoints/checkpoint1/OPTIONS-000003
 open-dir: checkpoints/checkpoint1
-create: checkpoints/checkpoint1/marker.format-version.000001.016
-sync-data: checkpoints/checkpoint1/marker.format-version.000001.016
-close: checkpoints/checkpoint1/marker.format-version.000001.016
+create: checkpoints/checkpoint1/marker.format-version.000001.017
+sync-data: checkpoints/checkpoint1/marker.format-version.000001.017
+close: checkpoints/checkpoint1/marker.format-version.000001.017
 sync: checkpoints/checkpoint1
 close: checkpoints/checkpoint1
 link: db/000005.sst -> checkpoints/checkpoint1/000005.sst
@@ -127,9 +131,9 @@ close: checkpoints
 open-dir: checkpoints/checkpoint2
 link: db/OPTIONS-000003 -> checkpoints/checkpoint2/OPTIONS-000003
 open-dir: checkpoints/checkpoint2
-create: checkpoints/checkpoint2/marker.format-version.000001.016
-sync-data: checkpoints/checkpoint2/marker.format-version.000001.016
-close: checkpoints/checkpoint2/marker.format-version.000001.016
+create: checkpoints/checkpoint2/marker.format-version.000001.017
+sync-data: checkpoints/checkpoint2/marker.format-version.000001.017
+close: checkpoints/checkpoint2/marker.format-version.000001.017
 sync: checkpoints/checkpoint2
 close: checkpoints/checkpoint2
 link: db/000007.sst -> checkpoints/checkpoint2/000007.sst
@@ -162,9 +166,9 @@ close: checkpoints
 open-dir: checkpoints/checkpoint3
 link: db/OPTIONS-000003 -> checkpoints/checkpoint3/OPTIONS-000003
 open-dir: checkpoints/checkpoint3
-create: checkpoints/checkpoint3/marker.format-version.000001.016
-sync-data: checkpoints/checkpoint3/marker.format-version.000001.016
-close: checkpoints/checkpoint3/marker.format-version.000001.016
+create: checkpoints/checkpoint3/marker.format-version.000001.017
+sync-data: checkpoints/checkpoint3/marker.format-version.000001.017
+close: checkpoints/checkpoint3/marker.format-version.000001.017
 sync: checkpoints/checkpoint3
 close: checkpoints/checkpoint3
 link: db/000005.sst -> checkpoints/checkpoint3/000005.sst
@@ -254,7 +258,7 @@ list db
 LOCK
 MANIFEST-000001
 OPTIONS-000003
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
 list checkpoints/checkpoint1
@@ -264,7 +268,7 @@ list checkpoints/checkpoint1
 000007.sst
 MANIFEST-000001
 OPTIONS-000003
-marker.format-version.000001.016
+marker.format-version.000001.017
 marker.manifest.000001.MANIFEST-000001
 
 open checkpoints/checkpoint1 readonly
@@ -329,7 +333,7 @@ list checkpoints/checkpoint2
 000007.sst
 MANIFEST-000001
 OPTIONS-000003
-marker.format-version.000001.016
+marker.format-version.000001.017
 marker.manifest.000001.MANIFEST-000001
 
 open checkpoints/checkpoint2 readonly
@@ -369,7 +373,7 @@ list checkpoints/checkpoint3
 000007.sst
 MANIFEST-000001
 OPTIONS-000003
-marker.format-version.000001.016
+marker.format-version.000001.017
 marker.manifest.000001.MANIFEST-000001
 
 open checkpoints/checkpoint3 readonly
@@ -481,9 +485,9 @@ close: checkpoints
 open-dir: checkpoints/checkpoint4
 link: db/OPTIONS-000003 -> checkpoints/checkpoint4/OPTIONS-000003
 open-dir: checkpoints/checkpoint4
-create: checkpoints/checkpoint4/marker.format-version.000001.016
-sync-data: checkpoints/checkpoint4/marker.format-version.000001.016
-close: checkpoints/checkpoint4/marker.format-version.000001.016
+create: checkpoints/checkpoint4/marker.format-version.000001.017
+sync-data: checkpoints/checkpoint4/marker.format-version.000001.017
+close: checkpoints/checkpoint4/marker.format-version.000001.017
 sync: checkpoints/checkpoint4
 close: checkpoints/checkpoint4
 link: db/000010.sst -> checkpoints/checkpoint4/000010.sst
@@ -569,7 +573,7 @@ list db
 LOCK
 MANIFEST-000001
 OPTIONS-000003
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
 
@@ -584,9 +588,9 @@ close: checkpoints
 open-dir: checkpoints/checkpoint5
 link: db/OPTIONS-000003 -> checkpoints/checkpoint5/OPTIONS-000003
 open-dir: checkpoints/checkpoint5
-create: checkpoints/checkpoint5/marker.format-version.000001.016
-sync-data: checkpoints/checkpoint5/marker.format-version.000001.016
-close: checkpoints/checkpoint5/marker.format-version.000001.016
+create: checkpoints/checkpoint5/marker.format-version.000001.017
+sync-data: checkpoints/checkpoint5/marker.format-version.000001.017
+close: checkpoints/checkpoint5/marker.format-version.000001.017
 sync: checkpoints/checkpoint5
 close: checkpoints/checkpoint5
 link: db/000010.sst -> checkpoints/checkpoint5/000010.sst
@@ -675,9 +679,9 @@ close: checkpoints
 open-dir: checkpoints/checkpoint6
 link: db/OPTIONS-000003 -> checkpoints/checkpoint6/OPTIONS-000003
 open-dir: checkpoints/checkpoint6
-create: checkpoints/checkpoint6/marker.format-version.000001.016
-sync-data: checkpoints/checkpoint6/marker.format-version.000001.016
-close: checkpoints/checkpoint6/marker.format-version.000001.016
+create: checkpoints/checkpoint6/marker.format-version.000001.017
+sync-data: checkpoints/checkpoint6/marker.format-version.000001.017
+close: checkpoints/checkpoint6/marker.format-version.000001.017
 sync: checkpoints/checkpoint6
 close: checkpoints/checkpoint6
 link: db/000011.sst -> checkpoints/checkpoint6/000011.sst

testdata/event_listener

@@ -32,6 +32,11 @@ close: db/marker.format-version.000003.016
 remove: db/marker.format-version.000002.015
 sync: db
 upgraded to format version: 016
+create: db/marker.format-version.000004.017
+close: db/marker.format-version.000004.017
+remove: db/marker.format-version.000003.016
+sync: db
+upgraded to format version: 017
 create: db/temporary.000003.dbtmp
 sync: db/temporary.000003.dbtmp
 close: db/temporary.000003.dbtmp
@@ -343,9 +348,9 @@ close:
 open-dir: checkpoint
 link: db/OPTIONS-000003 -> checkpoint/OPTIONS-000003
 open-dir: checkpoint
-create: checkpoint/marker.format-version.000001.016
-sync-data: checkpoint/marker.format-version.000001.016
-close: checkpoint/marker.format-version.000001.016
+create: checkpoint/marker.format-version.000001.017
+sync-data: checkpoint/marker.format-version.000001.017
+close: checkpoint/marker.format-version.000001.017
 sync: checkpoint
 close: checkpoint
 link: db/000013.sst -> checkpoint/000013.sst

testdata/flushable_ingest

@@ -60,7 +60,7 @@ LOCK
 MANIFEST-000001
 OPTIONS-000003
 ext
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
 # Test basic WAL replay
@@ -81,7 +81,7 @@ LOCK
 MANIFEST-000001
 OPTIONS-000003
 ext
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
 open
@@ -389,7 +389,7 @@ LOCK
 MANIFEST-000001
 OPTIONS-000003
 ext
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
 close
@@ -409,7 +409,7 @@ LOCK
 MANIFEST-000001
 OPTIONS-000003
 ext
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
 open
@@ -441,7 +441,7 @@ MANIFEST-000001
 MANIFEST-000012
 OPTIONS-000013
 ext
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000002.MANIFEST-000012
 
 # Make sure that the new mutable memtable can accept writes.
@@ -584,7 +584,7 @@ LOCK
 MANIFEST-000001
 OPTIONS-000003
 ext
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
 close
@@ -603,7 +603,7 @@ MANIFEST-000001
 OPTIONS-000003
 ext
 ext1
-marker.format-version.000003.016
+marker.format-version.000004.017
 marker.manifest.000001.MANIFEST-000001
 
 ignoreSyncs false