diff --git a/format_major_version.go b/format_major_version.go index 27cef3028c..e0fa0c717a 100644 --- a/format_major_version.go +++ b/format_major_version.go @@ -159,6 +159,13 @@ const ( // sstable/format.go for details). ExperimentalFormatDeleteSizedAndObsolete + // FormatVirtualSSTables is a format major version that adds support for + // virtual sstables that can reference a sub-range of keys in an underlying + // physical sstables. This information is persisted through new, + // backward-incompatible fields in the Manifest, and therefore requires + // an FMV. + FormatVirtualSSTables + // internalFormatNewest holds the newest format major version, including // experimental ones excluded from the exported FormatNewest constant until // they've stabilized. Used in tests. @@ -184,7 +191,7 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat { return sstable.TableFormatPebblev2 case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted: return sstable.TableFormatPebblev3 - case ExperimentalFormatDeleteSizedAndObsolete: + case ExperimentalFormatDeleteSizedAndObsolete, FormatVirtualSSTables: return sstable.TableFormatPebblev4 default: panic(fmt.Sprintf("pebble: unsupported format major version: %s", v)) @@ -203,7 +210,7 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat { case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked, FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted, - ExperimentalFormatDeleteSizedAndObsolete: + ExperimentalFormatDeleteSizedAndObsolete, FormatVirtualSSTables: return sstable.TableFormatPebblev1 default: panic(fmt.Sprintf("pebble: unsupported format major version: %s", v)) @@ -339,6 +346,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ ExperimentalFormatDeleteSizedAndObsolete: func(d *DB) error { return d.finalizeFormatVersUpgrade(ExperimentalFormatDeleteSizedAndObsolete) }, + FormatVirtualSSTables: func(d *DB) error { + return d.finalizeFormatVersUpgrade(FormatVirtualSSTables) + }, } const formatVersionMarkerName = `format-version` diff --git a/format_major_version_test.go b/format_major_version_test.go index 571749d1c3..46f8e10ec2 100644 --- a/format_major_version_test.go +++ b/format_major_version_test.go @@ -64,6 +64,8 @@ func TestRatchetFormat(t *testing.T) { require.Equal(t, FormatPrePebblev1MarkedCompacted, d.FormatMajorVersion()) require.NoError(t, d.RatchetFormatMajorVersion(ExperimentalFormatDeleteSizedAndObsolete)) require.Equal(t, ExperimentalFormatDeleteSizedAndObsolete, d.FormatMajorVersion()) + require.NoError(t, d.RatchetFormatMajorVersion(FormatVirtualSSTables)) + require.Equal(t, FormatVirtualSSTables, d.FormatMajorVersion()) require.NoError(t, d.Close()) @@ -228,6 +230,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) { FormatFlushableIngest: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, FormatPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, ExperimentalFormatDeleteSizedAndObsolete: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, + FormatVirtualSSTables: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, } // Valid versions. diff --git a/open_test.go b/open_test.go index a13e81fae6..ee6ede0ff5 100644 --- a/open_test.go +++ b/open_test.go @@ -197,7 +197,7 @@ func TestNewDBFilenames(t *testing.T) { "LOCK", "MANIFEST-000001", "OPTIONS-000003", - "marker.format-version.000014.015", + "marker.format-version.000015.016", "marker.manifest.000001.MANIFEST-000001", }, } diff --git a/testdata/checkpoint b/testdata/checkpoint index 9c38a7c932..518baca54f 100644 --- a/testdata/checkpoint +++ b/testdata/checkpoint @@ -81,6 +81,10 @@ create: db/marker.format-version.000014.015 close: db/marker.format-version.000014.015 remove: db/marker.format-version.000013.014 sync: db +create: db/marker.format-version.000015.016 +close: db/marker.format-version.000015.016 +remove: db/marker.format-version.000014.015 +sync: db create: db/temporary.000003.dbtmp sync: db/temporary.000003.dbtmp close: db/temporary.000003.dbtmp @@ -143,9 +147,9 @@ close: open-dir: checkpoints/checkpoint1 link: db/OPTIONS-000003 -> checkpoints/checkpoint1/OPTIONS-000003 open-dir: checkpoints/checkpoint1 -create: checkpoints/checkpoint1/marker.format-version.000001.015 -sync-data: checkpoints/checkpoint1/marker.format-version.000001.015 -close: checkpoints/checkpoint1/marker.format-version.000001.015 +create: checkpoints/checkpoint1/marker.format-version.000001.016 +sync-data: checkpoints/checkpoint1/marker.format-version.000001.016 +close: checkpoints/checkpoint1/marker.format-version.000001.016 sync: checkpoints/checkpoint1 close: checkpoints/checkpoint1 link: db/000005.sst -> checkpoints/checkpoint1/000005.sst @@ -183,9 +187,9 @@ close: checkpoints open-dir: checkpoints/checkpoint2 link: db/OPTIONS-000003 -> checkpoints/checkpoint2/OPTIONS-000003 open-dir: checkpoints/checkpoint2 -create: checkpoints/checkpoint2/marker.format-version.000001.015 -sync-data: checkpoints/checkpoint2/marker.format-version.000001.015 -close: checkpoints/checkpoint2/marker.format-version.000001.015 +create: checkpoints/checkpoint2/marker.format-version.000001.016 +sync-data: checkpoints/checkpoint2/marker.format-version.000001.016 +close: checkpoints/checkpoint2/marker.format-version.000001.016 sync: checkpoints/checkpoint2 close: checkpoints/checkpoint2 link: db/000007.sst -> checkpoints/checkpoint2/000007.sst @@ -218,9 +222,9 @@ close: checkpoints open-dir: checkpoints/checkpoint3 link: db/OPTIONS-000003 -> checkpoints/checkpoint3/OPTIONS-000003 open-dir: checkpoints/checkpoint3 -create: checkpoints/checkpoint3/marker.format-version.000001.015 -sync-data: checkpoints/checkpoint3/marker.format-version.000001.015 -close: checkpoints/checkpoint3/marker.format-version.000001.015 +create: checkpoints/checkpoint3/marker.format-version.000001.016 +sync-data: checkpoints/checkpoint3/marker.format-version.000001.016 +close: checkpoints/checkpoint3/marker.format-version.000001.016 sync: checkpoints/checkpoint3 close: checkpoints/checkpoint3 link: db/000005.sst -> checkpoints/checkpoint3/000005.sst @@ -311,7 +315,7 @@ CURRENT LOCK MANIFEST-000001 OPTIONS-000003 -marker.format-version.000014.015 +marker.format-version.000015.016 marker.manifest.000001.MANIFEST-000001 list checkpoints/checkpoint1 @@ -321,7 +325,7 @@ list checkpoints/checkpoint1 000007.sst MANIFEST-000001 OPTIONS-000003 -marker.format-version.000001.015 +marker.format-version.000001.016 marker.manifest.000001.MANIFEST-000001 open checkpoints/checkpoint1 readonly @@ -386,7 +390,7 @@ list checkpoints/checkpoint2 000007.sst MANIFEST-000001 OPTIONS-000003 -marker.format-version.000001.015 +marker.format-version.000001.016 marker.manifest.000001.MANIFEST-000001 open checkpoints/checkpoint2 readonly @@ -426,7 +430,7 @@ list checkpoints/checkpoint3 000007.sst MANIFEST-000001 OPTIONS-000003 -marker.format-version.000001.015 +marker.format-version.000001.016 marker.manifest.000001.MANIFEST-000001 open checkpoints/checkpoint3 readonly diff --git a/testdata/event_listener b/testdata/event_listener index 172ee775d8..934f091533 100644 --- a/testdata/event_listener +++ b/testdata/event_listener @@ -99,6 +99,11 @@ close: db/marker.format-version.000014.015 remove: db/marker.format-version.000013.014 sync: db upgraded to format version: 015 +create: db/marker.format-version.000015.016 +close: db/marker.format-version.000015.016 +remove: db/marker.format-version.000014.015 +sync: db +upgraded to format version: 016 create: db/temporary.000003.dbtmp sync: db/temporary.000003.dbtmp close: db/temporary.000003.dbtmp @@ -400,9 +405,9 @@ close: open-dir: checkpoint link: db/OPTIONS-000003 -> checkpoint/OPTIONS-000003 open-dir: checkpoint -create: checkpoint/marker.format-version.000001.015 -sync-data: checkpoint/marker.format-version.000001.015 -close: checkpoint/marker.format-version.000001.015 +create: checkpoint/marker.format-version.000001.016 +sync-data: checkpoint/marker.format-version.000001.016 +close: checkpoint/marker.format-version.000001.016 sync: checkpoint close: checkpoint link: db/000013.sst -> checkpoint/000013.sst diff --git a/testdata/flushable_ingest b/testdata/flushable_ingest index dffb8f77a1..ea20491fcd 100644 --- a/testdata/flushable_ingest +++ b/testdata/flushable_ingest @@ -61,7 +61,7 @@ LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000014.015 +marker.format-version.000015.016 marker.manifest.000001.MANIFEST-000001 # Test basic WAL replay @@ -83,7 +83,7 @@ LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000014.015 +marker.format-version.000015.016 marker.manifest.000001.MANIFEST-000001 open @@ -392,7 +392,7 @@ LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000014.015 +marker.format-version.000015.016 marker.manifest.000001.MANIFEST-000001 close @@ -413,7 +413,7 @@ LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000014.015 +marker.format-version.000015.016 marker.manifest.000001.MANIFEST-000001 open @@ -446,7 +446,7 @@ MANIFEST-000001 MANIFEST-000012 OPTIONS-000013 ext -marker.format-version.000014.015 +marker.format-version.000015.016 marker.manifest.000002.MANIFEST-000012 # Make sure that the new mutable memtable can accept writes. @@ -590,7 +590,7 @@ LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000014.015 +marker.format-version.000015.016 marker.manifest.000001.MANIFEST-000001 close @@ -610,7 +610,7 @@ MANIFEST-000001 OPTIONS-000003 ext ext1 -marker.format-version.000014.015 +marker.format-version.000015.016 marker.manifest.000001.MANIFEST-000001 ignoreSyncs false diff --git a/version_set.go b/version_set.go index 61e7d7eb49..babd2bf2f0 100644 --- a/version_set.go +++ b/version_set.go @@ -490,6 +490,9 @@ func (vs *versionSet) logAndApply( defer vs.mu.Lock() var err error + if vs.opts.FormatMajorVersion < FormatVirtualSSTables && len(ve.CreatedBackingTables) > 0 { + return errors.AssertionFailedf("MANIFEST cannot contain virtual sstable records due to format major version") + } newVersion, zombies, err = manifest.AccumulateIncompleteAndApplySingleVE( ve, currentVersion, vs.cmp, vs.opts.Comparer.FormatKey, vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate, @@ -516,7 +519,7 @@ func (vs *versionSet) logAndApply( return errors.Wrap(err, "MANIFEST next record write failed") } - // NB: Any error from this point on is considered fatal as we don't now if + // NB: Any error from this point on is considered fatal as we don't know if // the MANIFEST write occurred or not. Trying to determine that is // fraught. Instead we rely on the standard recovery mechanism run when a // database is open. In particular, that mechanism generates a new MANIFEST diff --git a/version_set_test.go b/version_set_test.go index 21e46bc135..98e05b17db 100644 --- a/version_set_test.go +++ b/version_set_test.go @@ -41,6 +41,7 @@ func TestLatestRefCounting(t *testing.T) { FS: mem, MaxManifestFileSize: 1, DisableAutomaticCompactions: true, + FormatMajorVersion: FormatVirtualSSTables, } d, err := Open("", opts) require.NoError(t, err) @@ -217,6 +218,7 @@ func TestVirtualSSTableManifestReplay(t *testing.T) { require.NoError(t, mem.MkdirAll("ext", 0755)) opts := &Options{ + FormatMajorVersion: FormatVirtualSSTables, FS: mem, MaxManifestFileSize: 1, DisableAutomaticCompactions: true,