diff --git a/README.md b/README.md index c09e45d802c..683e06ece8c 100644 --- a/README.md +++ b/README.md @@ -86,17 +86,22 @@ differences. ## RocksDB Compatibility -Pebble strives for forward compatibility with RocksDB 6.2.1 (the latest -version of RocksDB used by CockroachDB). Forward compatibility means -that a DB generated by RocksDB can be used by Pebble. Currently, Pebble -provides bidirectional compatibility with RocksDB (a Pebble generated DB -can be used by RocksDB) when using its FormatMostCompatible format. New -functionality that is backwards incompatible is gated behind new format -major versions. In general, Pebble only provides compatibility with the -subset of functionality and configuration used by CockroachDB. The scope -of RocksDB functionality and configuration is too large to adequately -test and document all the incompatibilities. The list below contains -known incompatibilities. +Pebble strives for forward compatibility with RocksDB 6.2.1 (the latest version +of RocksDB used by CockroachDB). Forward compatibility means that a DB generated +by RocksDB 6.2.1 can be upgraded for use by Pebble. Pebble versions in the `v1` +series may open DBs generated by RocksDB 6.2.1. Since its introduction, Pebble +has adopted various backwards-incompatible format changes that are gated behind +new 'format major versions'. The Pebble `master` branch does not support opening +DBs generated by RocksDB. DBs generated by RocksDB may only be used with recent +versions of Pebble after migrating them through format major version upgrades +using previous versions of Pebble. See the below section of format major +versions. + +Even the RocksDB-compatible versions of Pebble only provide compatibility with +the subset of functionality and configuration used by CockroachDB. The scope of +RocksDB functionality and configuration is too large to adequately test and +document all the incompatibilities. The list below contains known +incompatibilities. * Pebble's use of WAL recycling is only compatible with RocksDB's `kTolerateCorruptedTailRecords` WAL recovery mode. Older versions of @@ -119,9 +124,14 @@ known incompatibilities. Over time Pebble has introduced new physical file formats. Backwards incompatible changes are made through the introduction of 'format major -versions'. By default, when Pebble opens a database, it defaults to -`FormatMostCompatible`. This version is bi-directionally compatible with RocksDB -6.2.1 (with the caveats described above). +versions'. By default, when Pebble opens a database, it defaults to the lowest +supported version. In `v1`, this is `FormatMostCompatible`, which is +bi-directionally compatible with RocksDB 6.2.1 (with the caveats described +above). + +Databases created by RocksDB or Pebble versions `v1` and earlier must be upgraded +to a compatible format major version before running newer Pebble versions. Newer +Pebble versions will refuse to open databases in no longer supported formats. To opt into new formats, a user may set `FormatMajorVersion` on the [`Options`](https://pkg.go.dev/github.com/cockroachdb/pebble#Options) @@ -132,24 +142,25 @@ upgrade the format major version at runtime using Format major version upgrades are permanent; There is no option to return to an earlier format. -The table below outlines the history of format major versions: - -| Name | Value | Migration | -|------------------------------------|-------|------------| -| FormatMostCompatible | 1 | No | -| FormatVersioned | 3 | No | -| FormatSetWithDelete | 4 | No | -| FormatBlockPropertyCollector | 5 | No | -| FormatSplitUserKeysMarked | 6 | Background | -| FormatSplitUserKeysMarkedCompacted | 7 | Blocking | -| FormatRangeKeys | 8 | No | -| FormatMinTableFormatPebblev1 | 9 | No | -| FormatPrePebblev1Marked | 10 | Background | -| FormatSSTableValueBlocks | 12 | No | -| FormatFlushableIngest | 13 | No | -| FormatPrePebblev1MarkedCompacted | 14 | Blocking | -| FormatDeleteSizedAndObsolete | 15 | No | -| FormatVirtualSSTables | 16 | No | +The table below outlines the history of format major versions, along with what +range of Pebble versions support that format. + +| Name | Value | Migration | Pebble support | +|------------------------------------|-------|------------|----------------| +| FormatMostCompatible | 1 | No | v1 | +| FormatVersioned | 3 | No | v1 | +| FormatSetWithDelete | 4 | No | v1 | +| FormatBlockPropertyCollector | 5 | No | v1 | +| FormatSplitUserKeysMarked | 6 | Background | v1 | +| FormatSplitUserKeysMarkedCompacted | 7 | Blocking | v1 | +| FormatRangeKeys | 8 | No | v1 | +| FormatMinTableFormatPebblev1 | 9 | No | v1 | +| FormatPrePebblev1Marked | 10 | Background | v1 | +| FormatSSTableValueBlocks | 12 | No | v1 | +| FormatFlushableIngest | 13 | No | v1, master | +| FormatPrePebblev1MarkedCompacted | 14 | Blocking | v1, master | +| FormatDeleteSizedAndObsolete | 15 | No | v1, master | +| FormatVirtualSSTables | 16 | No | v1, master | Upgrading to a format major version with 'Background' in the migration column may trigger background activity to rewrite physical file @@ -172,7 +183,6 @@ versions for CockroachDB releases. | 22.2 | FormatMostCompatible | FormatPrePebblev1Marked | | 23.1 | FormatSplitUserKeysMarkedCompacted | FormatFlushableIngest | | 23.2 | FormatSplitUserKeysMarkedCompacted | FormatVirtualSSTables | -| 24.1 plan | FormatSSTableValueBlocks | | ## Pedigree diff --git a/batch.go b/batch.go index ab7fdfd8c7b..02eec226912 100644 --- a/batch.go +++ b/batch.go @@ -484,9 +484,6 @@ func (b *Batch) refreshMemTableSize() error { } b.memTableSize += memTableEntrySize(len(key), len(value)) } - if b.countRangeKeys > 0 && b.minimumFormatMajorVersion < FormatRangeKeys { - b.minimumFormatMajorVersion = FormatRangeKeys - } return nil } @@ -958,9 +955,6 @@ func (b *Batch) rangeKeySetDeferred(startLen, internalValueLen int) *DeferredBat func (b *Batch) incrementRangeKeysCount() { b.countRangeKeys++ - if b.minimumFormatMajorVersion < FormatRangeKeys { - b.minimumFormatMajorVersion = FormatRangeKeys - } if b.index != nil { b.rangeKeys = nil b.rangeKeysSeqNum = 0 diff --git a/checkpoint.go b/checkpoint.go index f321c01ec3d..00ee6c56592 100644 --- a/checkpoint.go +++ b/checkpoint.go @@ -411,17 +411,12 @@ func (d *DB) writeCheckpointManifest( return err } - // Recent format versions use an atomic marker for setting the - // active manifest. Older versions use the CURRENT file. The - // setCurrentFunc function will return a closure that will - // take the appropriate action for the database's format - // version. var manifestMarker *atomicfs.Marker manifestMarker, _, err := atomicfs.LocateMarker(fs, destDirPath, manifestMarkerName) if err != nil { return err } - if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum); err != nil { + if err := manifestMarker.Move(base.MakeFilename(fileTypeManifest, manifestFileNum)); err != nil { return err } return manifestMarker.Close() diff --git a/compaction.go b/compaction.go index 784d5a28055..8144b16807f 100644 --- a/compaction.go +++ b/compaction.go @@ -1277,22 +1277,16 @@ func (c *compaction) newInputIter( newIters tableNewIters, newRangeKeyIter keyspan.TableNewSpanIter, snapshots []uint64, ) (_ internalIterator, retErr error) { // Validate the ordering of compaction input files for defense in depth. - // TODO(jackson): Some of the CheckOrdering calls may be adapted to pass - // ProhibitSplitUserKeys if we thread the active format major version in. Or - // if we remove support for earlier FMVs, we can remove the parameter - // altogether. if len(c.flushing) == 0 { if c.startLevel.level >= 0 { err := manifest.CheckOrdering(c.cmp, c.formatKey, - manifest.Level(c.startLevel.level), c.startLevel.files.Iter(), - manifest.AllowSplitUserKeys) + manifest.Level(c.startLevel.level), c.startLevel.files.Iter()) if err != nil { return nil, err } } err := manifest.CheckOrdering(c.cmp, c.formatKey, - manifest.Level(c.outputLevel.level), c.outputLevel.files.Iter(), - manifest.AllowSplitUserKeys) + manifest.Level(c.outputLevel.level), c.outputLevel.files.Iter()) if err != nil { return nil, err } @@ -1302,9 +1296,7 @@ func (c *compaction) newInputIter( } for _, info := range c.startLevel.l0SublevelInfo { err := manifest.CheckOrdering(c.cmp, c.formatKey, - info.sublevel, info.Iter(), - // NB: L0 sublevels have never allowed split user keys. - manifest.ProhibitSplitUserKeys) + info.sublevel, info.Iter()) if err != nil { return nil, err } @@ -1316,8 +1308,7 @@ func (c *compaction) newInputIter( } interLevel := c.extraLevels[0] err := manifest.CheckOrdering(c.cmp, c.formatKey, - manifest.Level(interLevel.level), interLevel.files.Iter(), - manifest.AllowSplitUserKeys) + manifest.Level(interLevel.level), interLevel.files.Iter()) if err != nil { return nil, err } @@ -3096,10 +3087,6 @@ func (d *DB) runCompaction( } writerOpts := d.opts.MakeWriterOptions(c.outputLevel.level, tableFormat) - if formatVers < FormatBlockPropertyCollector { - // Cannot yet write block properties. - writerOpts.BlockPropertyCollectors = nil - } // prevPointKey is a sstable.WriterOption that provides access to // the last point key written to a writer's sstable. When a new diff --git a/compaction_iter.go b/compaction_iter.go index cc753a6ac43..6d22476520e 100644 --- a/compaction_iter.go +++ b/compaction_iter.go @@ -728,12 +728,9 @@ func (i *compactionIter) setNext() { i.valid = true i.maybeZeroSeqnum(i.curSnapshotIdx) - // There are two cases where we can early return and skip the remaining + // If this key is already a SETWITHDEL we can early return and skip the remaining // records in the stripe: - // - If the DB does not SETWITHDEL. - // - If this key is already a SETWITHDEL. - if i.formatVersion < FormatSetWithDelete || - i.iterKey.Kind() == InternalKeyKindSetWithDelete { + if i.iterKey.Kind() == InternalKeyKindSetWithDelete { i.skip = true return } diff --git a/compaction_iter_test.go b/compaction_iter_test.go index 07c489cab59..c82e402deb1 100644 --- a/compaction_iter_test.go +++ b/compaction_iter_test.go @@ -87,18 +87,6 @@ func TestCompactionIter(t *testing.T) { var allowZeroSeqnum bool var interleavingIter *keyspan.InterleavingIter - // The input to the data-driven test is dependent on the format major - // version we are testing against. - fileFunc := func(formatVersion FormatMajorVersion) string { - if formatVersion < FormatSetWithDelete { - return "testdata/compaction_iter" - } - if formatVersion < FormatDeleteSizedAndObsolete { - return "testdata/compaction_iter_set_with_del" - } - return "testdata/compaction_iter_delete_sized" - } - newIter := func(formatVersion FormatMajorVersion) *compactionIter { // To adhere to the existing assumption that range deletion blocks in // SSTables are not released while iterating, and therefore not @@ -141,7 +129,7 @@ func TestCompactionIter(t *testing.T) { } runTest := func(t *testing.T, formatVersion FormatMajorVersion) { - datadriven.RunTest(t, fileFunc(formatVersion), func(t *testing.T, d *datadriven.TestData) string { + datadriven.RunTest(t, "testdata/compaction_iter_delete_sized", func(t *testing.T, d *datadriven.TestData) string { switch d.Cmd { case "define": merge = nil @@ -310,9 +298,7 @@ func TestCompactionIter(t *testing.T) { // Rather than testing against all format version, we test against the // significant boundaries. formatVersions := []FormatMajorVersion{ - FormatMostCompatible, - FormatSetWithDelete - 1, - FormatSetWithDelete, + FormatMinSupported, internalFormatNewest, } for _, formatVersion := range formatVersions { diff --git a/compaction_test.go b/compaction_test.go index 519b327f8a2..2ec0571f606 100644 --- a/compaction_test.go +++ b/compaction_test.go @@ -1575,63 +1575,41 @@ func TestManualCompaction(t *testing.T) { testCases := []struct { testData string - minVersion FormatMajorVersion - maxVersion FormatMajorVersion // inclusive + minVersion FormatMajorVersion // inclusive, FormatMinSupported if unspecified. + maxVersion FormatMajorVersion // inclusive, internalFormatNewest if unspecified. verbose bool }{ { - testData: "testdata/manual_compaction", - minVersion: FormatMostCompatible, - maxVersion: FormatSetWithDelete - 1, + testData: "testdata/singledel_manual_compaction_set_with_del", }, { - testData: "testdata/manual_compaction_set_with_del", - minVersion: FormatBlockPropertyCollector, - // This test exercises split user keys. - maxVersion: FormatSplitUserKeysMarkedCompacted - 1, - }, - { - testData: "testdata/singledel_manual_compaction", - minVersion: FormatMostCompatible, - maxVersion: FormatSetWithDelete - 1, - }, - { - testData: "testdata/singledel_manual_compaction_set_with_del", - minVersion: FormatSetWithDelete, - maxVersion: internalFormatNewest, - }, - { - testData: "testdata/manual_compaction_range_keys", - minVersion: FormatRangeKeys, - maxVersion: internalFormatNewest, - verbose: true, - }, - { - testData: "testdata/manual_compaction_file_boundaries", - minVersion: FormatBlockPropertyCollector, - // This test exercises split user keys. - maxVersion: FormatSplitUserKeysMarkedCompacted - 1, + testData: "testdata/manual_compaction_range_keys", + verbose: true, }, { testData: "testdata/manual_compaction_file_boundaries_delsized", minVersion: FormatDeleteSizedAndObsolete, - maxVersion: internalFormatNewest, }, { testData: "testdata/manual_compaction_set_with_del_sstable_Pebblev4", minVersion: FormatDeleteSizedAndObsolete, - maxVersion: internalFormatNewest, }, { - testData: "testdata/manual_compaction_multilevel", - minVersion: FormatMostCompatible, - maxVersion: internalFormatNewest, + testData: "testdata/manual_compaction_multilevel", }, } for _, tc := range testCases { t.Run(tc.testData, func(t *testing.T) { - runTest(t, tc.testData, tc.minVersion, tc.maxVersion, tc.verbose) + minVersion, maxVersion := tc.minVersion, tc.maxVersion + if minVersion == 0 { + minVersion = FormatMinSupported + } + if maxVersion == 0 { + maxVersion = internalFormatNewest + } + + runTest(t, tc.testData, minVersion, maxVersion, tc.verbose) }) } } diff --git a/data_test.go b/data_test.go index 9f6260f7d06..059b5ecd962 100644 --- a/data_test.go +++ b/data_test.go @@ -504,10 +504,6 @@ func runBuildRemoteCmd(td *datadriven.TestData, d *DB, storage remote.Storage) e switch cmdArg.Key { case "format": switch cmdArg.Vals[0] { - case "leveldb": - tableFormat = sstable.TableFormatLevelDB - case "rocksdbv2": - tableFormat = sstable.TableFormatRocksDBv2 case "pebblev1": tableFormat = sstable.TableFormatPebblev1 case "pebblev2": @@ -594,10 +590,6 @@ func runBuildCmd(td *datadriven.TestData, d *DB, fs vfs.FS) error { switch cmdArg.Key { case "format": switch cmdArg.Vals[0] { - case "leveldb": - tableFormat = sstable.TableFormatLevelDB - case "rocksdbv2": - tableFormat = sstable.TableFormatRocksDBv2 case "pebblev1": tableFormat = sstable.TableFormatPebblev1 case "pebblev2": @@ -1304,37 +1296,6 @@ func runIngestExternalCmd(td *datadriven.TestData, d *DB, locator string) error return nil } -func runForceIngestCmd(td *datadriven.TestData, d *DB) error { - var paths []string - var level int - for _, arg := range td.CmdArgs { - switch arg.Key { - case "paths": - paths = append(paths, arg.Vals...) - case "level": - var err error - level, err = strconv.Atoi(arg.Vals[0]) - if err != nil { - return err - } - } - } - _, err := d.ingest(paths, func( - tableNewIters, - keyspan.TableNewSpanIter, - IterOptions, - *Comparer, - *version, - int, - map[*compaction]struct{}, - *fileMetadata, - bool, - ) (int, *fileMetadata, error) { - return level, nil, nil - }, nil /* shared */, KeyRange{}, nil /* external */) - return err -} - func runLSMCmd(td *datadriven.TestData, d *DB) string { d.mu.Lock() defer d.mu.Unlock() diff --git a/db.go b/db.go index 4c88395ecd8..df0e20ec317 100644 --- a/db.go +++ b/db.go @@ -826,13 +826,11 @@ func (d *DB) applyInternal(batch *Batch, opts *WriteOptions, noSyncWait bool) er return errors.New("pebble: WAL disabled") } - if batch.minimumFormatMajorVersion != FormatMostCompatible { - if fmv := d.FormatMajorVersion(); fmv < batch.minimumFormatMajorVersion { - panic(fmt.Sprintf( - "pebble: batch requires at least format major version %d (current: %d)", - batch.minimumFormatMajorVersion, fmv, - )) - } + if fmv := d.FormatMajorVersion(); fmv < batch.minimumFormatMajorVersion { + panic(fmt.Sprintf( + "pebble: batch requires at least format major version %d (current: %d)", + batch.minimumFormatMajorVersion, fmv, + )) } if batch.countRangeKeys > 0 { @@ -1034,14 +1032,6 @@ func (d *DB) newIter( panic(err) } seqNum := internalOpts.snapshot.seqNum - if o.rangeKeys() { - if d.FormatMajorVersion() < FormatRangeKeys { - panic(fmt.Sprintf( - "pebble: range keys require at least format major version %d (current: %d)", - FormatRangeKeys, d.FormatMajorVersion(), - )) - } - } if o != nil && o.RangeKeyMasking.Suffix != nil && o.KeyTypes != IterKeyTypePointsAndRanges { panic("pebble: range key masking requires IterKeyTypePointsAndRanges") } diff --git a/db_test.go b/db_test.go index 631753df923..d5e8a61f755 100644 --- a/db_test.go +++ b/db_test.go @@ -76,6 +76,7 @@ func TestTry(t *testing.T) { } func TestBasicReads(t *testing.T) { + t.Skip("TODO") testCases := []struct { dirname string wantMap map[string]string diff --git a/error_test.go b/error_test.go index 82af4a49a64..b73f48cf2a9 100644 --- a/error_test.go +++ b/error_test.go @@ -142,14 +142,17 @@ func TestErrors(t *testing.T) { t.Logf("success %d\n", i) break } - errorCounts[err.Error()]++ + errMsg := err.Error() + if !strings.Contains(errMsg, "injected error") { + t.Fatalf("unexpected errors: %v", err) + } + errorCounts[errMsg]++ } expectedErrors := []string{ "fatal: MANIFEST flush failed: injected error", "fatal: MANIFEST sync failed: injected error", "fatal: MANIFEST set current failed: injected error", - "fatal: MANIFEST dirsync failed: injected error", } for _, expected := range expectedErrors { if errorCounts[expected] == 0 { @@ -193,21 +196,12 @@ func TestRequireReadError(t *testing.T) { require.NoError(t, d.DeleteRange(key1, key2, nil)) require.NoError(t, d.Set(key1, value, nil)) require.NoError(t, d.Flush()) - if formatVersion < FormatSetWithDelete { - expectLSM(` -0.0: - 000007:[a1#13,SET-a2#inf,RANGEDEL] -6: - 000005:[a1#10,SET-a2#11,SET] -`, d, t) - } else { - expectLSM(` + expectLSM(` 0.0: 000007:[a1#13,SETWITHDEL-a2#inf,RANGEDEL] 6: 000005:[a1#10,SET-a2#11,SET] `, d, t) - } // Now perform foreground ops with error injection enabled. ii.Store(index) @@ -246,7 +240,7 @@ func TestRequireReadError(t *testing.T) { return nil } - versions := []FormatMajorVersion{FormatMostCompatible, FormatSetWithDelete} + versions := []FormatMajorVersion{FormatMinSupported, internalFormatNewest} for _, version := range versions { t.Run(fmt.Sprintf("version-%s", version), func(t *testing.T) { for i := int32(0); ; i++ { @@ -296,22 +290,12 @@ func TestCorruptReadError(t *testing.T) { require.NoError(t, d.DeleteRange(key1, key2, nil)) require.NoError(t, d.Set(key1, value, nil)) require.NoError(t, d.Flush()) - if formatVersion < FormatSetWithDelete { - expectLSM(` -0.0: - 000007:[a1#13,SET-a2#inf,RANGEDEL] -6: - 000005:[a1#10,SET-a2#11,SET] -`, d, t) - - } else { - expectLSM(` + expectLSM(` 0.0: 000007:[a1#13,SETWITHDEL-a2#inf,RANGEDEL] 6: 000005:[a1#10,SET-a2#11,SET] `, d, t) - } // Now perform foreground ops with corruption injection enabled. fs.index.Store(index) @@ -349,7 +333,7 @@ func TestCorruptReadError(t *testing.T) { } return nil } - versions := []FormatMajorVersion{FormatMostCompatible, FormatSetWithDelete} + versions := []FormatMajorVersion{FormatMinSupported, internalFormatNewest} for _, version := range versions { t.Run(fmt.Sprintf("version-%s", version), func(t *testing.T) { for i := int32(0); ; i++ { diff --git a/external_iterator_test.go b/external_iterator_test.go index 77afd4dcc74..75b7acf6b97 100644 --- a/external_iterator_test.go +++ b/external_iterator_test.go @@ -27,10 +27,10 @@ import ( func TestExternalIterator(t *testing.T) { mem := vfs.NewMem() o := &Options{ - FS: mem, - Comparer: testkeys.Comparer, - FormatMajorVersion: FormatRangeKeys, + FS: mem, + Comparer: testkeys.Comparer, } + o.testingRandomized(t) o.EnsureDefaults() d, err := Open("", o) require.NoError(t, err) @@ -80,10 +80,10 @@ func TestExternalIterator(t *testing.T) { func TestSimpleLevelIter(t *testing.T) { mem := vfs.NewMem() o := &Options{ - FS: mem, - Comparer: testkeys.Comparer, - FormatMajorVersion: FormatRangeKeys, + FS: mem, + Comparer: testkeys.Comparer, } + o.testingRandomized(t) o.EnsureDefaults() d, err := Open("", o) require.NoError(t, err) diff --git a/filenames.go b/filenames.go index 07d74c87d36..86e7ed1d370 100644 --- a/filenames.go +++ b/filenames.go @@ -4,12 +4,7 @@ package pebble -import ( - "fmt" - - "github.com/cockroachdb/pebble/internal/base" - "github.com/cockroachdb/pebble/vfs" -) +import "github.com/cockroachdb/pebble/internal/base" type fileType = base.FileType @@ -21,34 +16,7 @@ const ( fileTypeLock = base.FileTypeLock fileTypeTable = base.FileTypeTable fileTypeManifest = base.FileTypeManifest - fileTypeCurrent = base.FileTypeCurrent fileTypeOptions = base.FileTypeOptions fileTypeTemp = base.FileTypeTemp fileTypeOldTemp = base.FileTypeOldTemp ) - -// setCurrentFile sets the CURRENT file to point to the manifest with -// provided file number. -// -// NB: This is a low-level routine and typically not what you want to -// use. Newer versions of Pebble running newer format major versions do -// not use the CURRENT file. See setCurrentFunc in version_set.go. -func setCurrentFile(dirname string, fs vfs.FS, fileNum base.DiskFileNum) error { - newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum) - oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum) - fs.Remove(oldFilename) - f, err := fs.Create(oldFilename) - if err != nil { - return err - } - if _, err := fmt.Fprintf(f, "MANIFEST-%s\n", fileNum); err != nil { - return err - } - if err := f.Sync(); err != nil { - return err - } - if err := f.Close(); err != nil { - return err - } - return fs.Rename(oldFilename, newFilename) -} diff --git a/format_major_version.go b/format_major_version.go index 89be1610df4..dc0b1b88700 100644 --- a/format_major_version.go +++ b/format_major_version.go @@ -9,7 +9,6 @@ import ( "strconv" "github.com/cockroachdb/errors" - "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" @@ -42,98 +41,104 @@ func (v FormatMajorVersion) String() string { return fmt.Sprintf("%03d", v) } -const ( - // 21.2 versions. - - // FormatDefault leaves the format version unspecified. The - // FormatDefault constant may be ratcheted upwards over time. - FormatDefault FormatMajorVersion = iota - // FormatMostCompatible maintains the most backwards compatibility, - // maintaining bi-directional compatibility with RocksDB 6.2.1 in - // the particular configuration described in the Pebble README. - FormatMostCompatible - // formatVersionedManifestMarker is the first - // backwards-incompatible change made to Pebble, introducing the - // format-version marker file for handling backwards-incompatible - // changes more broadly, and replacing the `CURRENT` file with a - // marker file. - // - // This format version is intended as an intermediary version state. - // It is deliberately unexported to discourage direct use of this - // format major version. Clients should use FormatVersioned which - // also ensures earlier versions of Pebble fail to open a database - // written in a future format major version. - formatVersionedManifestMarker - // FormatVersioned is a new format major version that replaces the - // old `CURRENT` file with a new 'marker' file scheme. Previous - // Pebble versions will be unable to open the database unless - // they're aware of format versions. - FormatVersioned - // FormatSetWithDelete is a format major version that introduces a new key - // kind, base.InternalKeyKindSetWithDelete. Previous Pebble versions will be - // unable to open this database. - FormatSetWithDelete - - // 22.1 versions. - - // FormatBlockPropertyCollector is a format major version that introduces - // BlockPropertyCollectors. - FormatBlockPropertyCollector - // FormatSplitUserKeysMarked is a format major version that guarantees that - // all files that share user keys with neighbors are marked for compaction - // in the manifest. Ratcheting to FormatSplitUserKeysMarked will block - // (without holding mutexes) until the scan of the LSM is complete and the - // manifest has been rotated. - FormatSplitUserKeysMarked - - // 22.2 versions. - - // FormatSplitUserKeysMarkedCompacted is a format major version that - // guarantees that all files explicitly marked for compaction in the manifest - // have been compacted. Combined with the FormatSplitUserKeysMarked format - // major version, this version guarantees that there are no user keys split - // across multiple files within a level L1+. Ratcheting to this format version - // will block (without holding mutexes) until all necessary compactions for - // files marked for compaction are complete. - FormatSplitUserKeysMarkedCompacted - // FormatRangeKeys is a format major version that introduces range keys. - FormatRangeKeys - // FormatMinTableFormatPebblev1 is a format major version that guarantees that - // tables created by or ingested into the DB at or above this format major - // version will have a table format version of at least Pebblev1 (Block - // Properties). - FormatMinTableFormatPebblev1 - // FormatPrePebblev1Marked is a format major version that guarantees that all - // sstables with a table format version pre-Pebblev1 (i.e. those that are - // guaranteed to not contain block properties) are marked for compaction in - // the manifest. Ratcheting to FormatPrePebblev1Marked will block (without - // holding mutexes) until the scan of the LSM is complete and the manifest has - // been rotated. - FormatPrePebblev1Marked - - // 23.1 versions. - - // formatUnusedPrePebblev1MarkedCompacted is an unused format major version. - // This format major version was originally intended to ship in the 23.1 - // release. It was later decided that this should be deferred until a - // subsequent release. The original ordering is preserved so as not to - // introduce breaking changes in Cockroach. - formatUnusedPrePebblev1MarkedCompacted - - // FormatSSTableValueBlocks is a format major version that adds support for - // storing values in value blocks in the sstable. Value block support is not - // necessarily enabled when writing sstables, when running with this format - // major version. - // - // WARNING: In development, so no production code should upgrade to this - // format, since a DB with this format major version will not actually - // interoperate correctly with another DB with the same format major - // version. This format major version is introduced so that tests can start - // being executed up to this version. Note that these tests succeed despite - // the incomplete support since they do not enable value blocks and use - // TableFormatPebblev2. - FormatSSTableValueBlocks +/* + Deprecated versions are retained in this comment for reference. + + // 21.2 versions. + + // FormatMostCompatible maintains the most backwards compatibility, + // maintaining bi-directional compatibility with RocksDB 6.2.1 in + // the particular configuration described in the Pebble README. + FormatMostCompatible + // formatVersionedManifestMarker is the first + // backwards-incompatible change made to Pebble, introducing the + // format-version marker file for handling backwards-incompatible + // changes more broadly, and replacing the `CURRENT` file with a + // marker file. + // + // This format version is intended as an intermediary version state. + // It is deliberately unexported to discourage direct use of this + // format major version. Clients should use FormatVersioned which + // also ensures earlier versions of Pebble fail to open a database + // written in a future format major version. + formatVersionedManifestMarker + + // FormatVersioned is a new format major version that replaces the + // old `CURRENT` file with a new 'marker' file scheme. Previous + // Pebble versions will be unable to open the database unless + // they're aware of format versions. + FormatVersioned + + // FormatSetWithDelete is a format major version that introduces a new key + // kind, base.InternalKeyKindSetWithDelete. Previous Pebble versions will be + // unable to open this database. + FormatSetWithDelete + + // 22.1 versions. + + // FormatBlockPropertyCollector is a format major version that introduces + // BlockPropertyCollectors. + FormatBlockPropertyCollector + + // FormatSplitUserKeysMarked is a format major version that guarantees that + // all files that share user keys with neighbors are marked for compaction + // in the manifest. Ratcheting to FormatSplitUserKeysMarked will block + // (without holding mutexes) until the scan of the LSM is complete and the + // manifest has been rotated. + // Deprecated. + FormatSplitUserKeysMarked + + // 22.2 versions. + + // FormatSplitUserKeysMarkedCompacted is a format major version that + // guarantees that all files explicitly marked for compaction in the manifest + // have been compacted. Combined with the FormatSplitUserKeysMarked format + // major version, this version guarantees that there are no user keys split + // across multiple files within a level L1+. Ratcheting to this format version + // will block (without holding mutexes) until all necessary compactions for + // files marked for compaction are complete. + // Deprecated. + FormatSplitUserKeysMarkedCompacted + + // FormatRangeKeys is a format major version that introduces range keys. + // Deprecated. + FormatRangeKeys + + // FormatMinTableFormatPebblev1 is a format major version that guarantees that + // tables created by or ingested into the DB at or above this format major + // version will have a table format version of at least Pebblev1 (Block + // Properties). + // Deprecated. + FormatMinTableFormatPebblev1 + + // FormatPrePebblev1Marked is a format major version that guarantees that all + // sstables with a table format version pre-Pebblev1 (i.e. those that are + // guaranteed to not contain block properties) are marked for compaction in + // the manifest. Ratcheting to FormatPrePebblev1Marked will block (without + // holding mutexes) until the scan of the LSM is complete and the manifest has + // been rotated. + // Deprecated. + FormatPrePebblev1Marked + + // 23.1 versions. + + // formatUnusedPrePebblev1MarkedCompacted is an unused format major version. + // This format major version was originally intended to ship in the 23.1 + // release. It was later decided that this should be deferred until a + // subsequent release. The original ordering is preserved so as not to + // introduce breaking changes in Cockroach. + formatUnusedPrePebblev1MarkedCompacted + + // FormatSSTableValueBlocks is a format major version that adds support for + // storing values in value blocks in the sstable. Value block support is not + // necessarily enabled when writing sstables, when running with this format + // major version. + FormatSSTableValueBlocks +*/ +const numDeprecatedVersions = 12 + +const ( // FormatFlushableIngest is a format major version that enables lazy // addition of ingested sstables into the LSM structure. When an ingest // overlaps with a memtable, a record of the ingest is written to the WAL @@ -143,7 +148,7 @@ const ( // // This feature is behind a format major version because it required // breaking changes to the WAL format. - FormatFlushableIngest + FormatFlushableIngest FormatMajorVersion = numDeprecatedVersions + 1 + iota // 23.2 versions. @@ -169,29 +174,40 @@ const ( // a format major version. FormatVirtualSSTables - // internalFormatNewest holds the newest format major version, including - // experimental ones excluded from the exported FormatNewest constant until - // they've stabilized. Used in tests. - internalFormatNewest FormatMajorVersion = iota - 1 + // -- Add new versions here -- + + // FormatNewest is the most recent format major version. + FormatNewest FormatMajorVersion = numDeprecatedVersions + iota - // FormatNewest always contains the most recent format major version. - FormatNewest FormatMajorVersion = internalFormatNewest + // Experimental versions, which are excluded by FormatNewest (but can be used + // in tests) can be defined here. + + // -- Add experimental versions here -- + + // internalFormatNewest is the most recent, possibly experimental format major + // version. + internalFormatNewest FormatMajorVersion = numDeprecatedVersions + iota - 1 ) +// FormatDefault leaves the format version unspecified. When used to create a +// new store, Pebble will choose the earliest format version it supports. +const FormatDefault FormatMajorVersion = 0 + +// FormatMinSupported is the minimum format version that is supported by this +// Pebble version. +const FormatMinSupported = FormatFlushableIngest + +// IsSupported returns true if the version is supported by the current Pebble +// version. +func (v FormatMajorVersion) IsSupported() bool { + return v == FormatDefault && v >= FormatMinSupported && v <= internalFormatNewest +} + // MaxTableFormat returns the maximum sstable.TableFormat that can be used at // this FormatMajorVersion. func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat { switch v { - case FormatDefault, FormatMostCompatible, formatVersionedManifestMarker, - FormatVersioned, FormatSetWithDelete: - return sstable.TableFormatRocksDBv2 - case FormatBlockPropertyCollector, FormatSplitUserKeysMarked, - FormatSplitUserKeysMarkedCompacted: - return sstable.TableFormatPebblev1 - case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked, - formatUnusedPrePebblev1MarkedCompacted: - return sstable.TableFormatPebblev2 - case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted: + case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted: return sstable.TableFormatPebblev3 case FormatDeleteSizedAndObsolete, FormatVirtualSSTables: return sstable.TableFormatPebblev4 @@ -204,14 +220,7 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat { // this FormatMajorVersion. func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat { switch v { - case FormatDefault, FormatMostCompatible, formatVersionedManifestMarker, - FormatVersioned, FormatSetWithDelete, FormatBlockPropertyCollector, - FormatSplitUserKeysMarked, FormatSplitUserKeysMarkedCompacted, - FormatRangeKeys: - return sstable.TableFormatLevelDB - case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked, - formatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks, - FormatFlushableIngest, FormatPrePebblev1MarkedCompacted, + case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted, FormatDeleteSizedAndObsolete, FormatVirtualSSTables: return sstable.TableFormatPebblev1 default: @@ -219,18 +228,6 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat { } } -// orderingInvariants returns an enum encoding the set of invariants that must -// hold within the receiver format major version. Invariants only get stricter -// as the format major version advances, so it is okay to retrieve the -// invariants from the current format major version and by the time the -// invariants are enforced, the format major version has advanced. -func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants { - if v < FormatSplitUserKeysMarkedCompacted { - return manifest.AllowSplitUserKeys - } - return manifest.ProhibitSplitUserKeys -} - // formatMajorVersionMigrations defines the migrations from one format // major version to the next. Each migration is defined as a closure // which will be invoked on the database before the new format major @@ -242,112 +239,7 @@ func (v FormatMajorVersion) orderingInvariants() manifest.OrderingInvariants { // panic if a migration returns a nil error but fails to finalize the // new format major version. var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ - FormatMostCompatible: func(d *DB) error { return nil }, - formatVersionedManifestMarker: func(d *DB) error { - // formatVersionedManifestMarker introduces the use of a marker - // file for pointing to the current MANIFEST file. - - // Lock the manifest. - d.mu.versions.logLock() - defer d.mu.versions.logUnlock() - - // Construct the filename of the currently active manifest and - // move the manifest marker to that filename. The marker is - // guaranteed to exist, because we unconditionally locate it - // during Open. - manifestFileNum := d.mu.versions.manifestFileNum - filename := base.MakeFilename(fileTypeManifest, manifestFileNum) - if err := d.mu.versions.manifestMarker.Move(filename); err != nil { - return errors.Wrap(err, "moving manifest marker") - } - - // Now that we have a manifest marker file in place and pointing - // to the current MANIFEST, finalize the upgrade. If we fail for - // some reason, a retry of this migration is guaranteed to again - // move the manifest marker file to the latest manifest. If - // we're unable to finalize the upgrade, a subsequent call to - // Open will ignore the manifest marker. - if err := d.finalizeFormatVersUpgrade(formatVersionedManifestMarker); err != nil { - return err - } - - // We've finalized the upgrade. All subsequent Open calls will - // ignore the CURRENT file and instead read the manifest marker. - // Before we unlock the manifest, we need to update versionSet - // to use the manifest marker on future rotations. - d.mu.versions.setCurrent = setCurrentFuncMarker( - d.mu.versions.manifestMarker, - d.mu.versions.fs, - d.mu.versions.dirname) - return nil - }, - // The FormatVersioned version is split into two, each with their - // own migration to ensure the post-migration cleanup happens even - // if there's a crash immediately after finalizing the version. Once - // a new format major version is finalized, its migration will never - // run again. Post-migration cleanup like the one in the migration - // below must be performed in a separate migration or every time the - // database opens. - FormatVersioned: func(d *DB) error { - // Replace the `CURRENT` file with one that points to the - // nonexistent `MANIFEST-000000` file. If an earlier Pebble - // version that does not know about format major versions - // attempts to open the database, it will error avoiding - // accidental corruption. - if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, base.FileNum(0).DiskFileNum()); err != nil { - return err - } - return d.finalizeFormatVersUpgrade(FormatVersioned) - }, - // As SetWithDelete is a new key kind, there is nothing to migrate. We can - // simply finalize the format version and we're done. - FormatSetWithDelete: func(d *DB) error { - return d.finalizeFormatVersUpgrade(FormatSetWithDelete) - }, - FormatBlockPropertyCollector: func(d *DB) error { - return d.finalizeFormatVersUpgrade(FormatBlockPropertyCollector) - }, - FormatSplitUserKeysMarked: func(d *DB) error { - // Mark any unmarked files with split-user keys. Note all format major - // versions migrations are invoked with DB.mu locked. - if err := d.markFilesLocked(markFilesWithSplitUserKeys(d.opts.Comparer.Equal)); err != nil { - return err - } - return d.finalizeFormatVersUpgrade(FormatSplitUserKeysMarked) - }, - FormatSplitUserKeysMarkedCompacted: func(d *DB) error { - // Before finalizing the format major version, rewrite any sstables - // still marked for compaction. Note all format major versions - // migrations are invoked with DB.mu locked. - if err := d.compactMarkedFilesLocked(); err != nil { - return err - } - return d.finalizeFormatVersUpgrade(FormatSplitUserKeysMarkedCompacted) - }, - FormatRangeKeys: func(d *DB) error { - return d.finalizeFormatVersUpgrade(FormatRangeKeys) - }, - FormatMinTableFormatPebblev1: func(d *DB) error { - return d.finalizeFormatVersUpgrade(FormatMinTableFormatPebblev1) - }, - FormatPrePebblev1Marked: func(d *DB) error { - // Mark any unmarked files that contain only table properties. Note all - // format major versions migrations are invoked with DB.mu locked. - if err := d.markFilesLocked(markFilesPrePebblev1(d.tableCache)); err != nil { - return err - } - return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked) - }, - formatUnusedPrePebblev1MarkedCompacted: func(d *DB) error { - // Intentional no-op. - return d.finalizeFormatVersUpgrade(formatUnusedPrePebblev1MarkedCompacted) - }, - FormatSSTableValueBlocks: func(d *DB) error { - return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks) - }, - FormatFlushableIngest: func(d *DB) error { - return d.finalizeFormatVersUpgrade(FormatFlushableIngest) - }, + FormatFlushableIngest: func(d *DB) error { return nil }, FormatPrePebblev1MarkedCompacted: func(d *DB) error { // Before finalizing the format major version, rewrite any sstables // still marked for compaction. Note all format major versions @@ -367,6 +259,12 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ const formatVersionMarkerName = `format-version` +// lookupFormatMajorVersion retrieves the format version from the format version +// marker file. +// +// If such a file does not exist, returns FormatDefault. Note that this case is +// only acceptable if we are creating a new store (we no longer support +// FormatMostCompatible which is the only one with no version marker file). func lookupFormatMajorVersion( fs vfs.FS, dirname string, ) (FormatMajorVersion, *atomicfs.Marker, error) { @@ -375,7 +273,7 @@ func lookupFormatMajorVersion( return 0, nil, err } if versString == "" { - return FormatMostCompatible, m, nil + return FormatDefault, m, nil } v, err := strconv.ParseUint(versString, 10, 64) if err != nil { @@ -386,7 +284,10 @@ func lookupFormatMajorVersion( return 0, nil, errors.Newf("pebble: default format major version should not persisted", vers) } if vers > internalFormatNewest { - return 0, nil, errors.Newf("pebble: database %q written in format major version %d", dirname, vers) + return 0, nil, errors.Newf("pebble: database %q written in unknown format major version %d", dirname, vers) + } + if vers < FormatMinSupported { + return 0, nil, errors.Newf("pebble: database %q written in format major version %d which is no longer supported", dirname, vers) } return vers, m, nil } @@ -456,11 +357,7 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro // // See formatMajorVersionMigrations. func (d *DB) finalizeFormatVersUpgrade(formatVers FormatMajorVersion) error { - // We use the marker to encode the active format version in the - // marker filename. Unlike other uses of the atomic marker, there is - // no file with the filename `formatVers.String()` on the - // filesystem. - if err := d.mu.formatVers.marker.Move(formatVers.String()); err != nil { + if err := d.writeFormatVersionMarker(formatVers); err != nil { return err } d.mu.formatVers.vers.Store(uint64(formatVers)) @@ -468,6 +365,14 @@ func (d *DB) finalizeFormatVersUpgrade(formatVers FormatMajorVersion) error { return nil } +func (d *DB) writeFormatVersionMarker(formatVers FormatMajorVersion) error { + // We use the marker to encode the active format version in the + // marker filename. Unlike other uses of the atomic marker, there is + // no file with the filename `formatVers.String()` on the + // filesystem. + return d.mu.formatVers.marker.Move(formatVers.String()) +} + // compactMarkedFilesLocked performs a migration that schedules rewrite // compactions to compact away any sstables marked for compaction. // compactMarkedFilesLocked is run while ratcheting the database's format major @@ -523,73 +428,11 @@ func (d *DB) compactMarkedFilesLocked() error { // level. type findFilesFunc func(v *version) (found bool, files [numLevels][]*fileMetadata, _ error) -// markFilesWithSplitUserKeys scans the LSM's levels 1 through 6 for adjacent -// files that contain the same user key. Such arrangements of files were -// permitted in RocksDB and in Pebble up to SHA a860bbad. -var markFilesWithSplitUserKeys = func(equal Equal) findFilesFunc { - return func(v *version) (found bool, files [numLevels][]*fileMetadata, _ error) { - // Files with split user keys are expected to be rare and performing key - // comparisons for every file within the LSM is expensive, so drop the - // database lock while scanning the file metadata. - for l := numLevels - 1; l > 0; l-- { - iter := v.Levels[l].Iter() - var prevFile *fileMetadata - var prevUserKey []byte - for f := iter.First(); f != nil; f = iter.Next() { - if prevUserKey != nil && equal(prevUserKey, f.Smallest.UserKey) { - // NB: We may append a file twice, once as prevFile and once - // as f. That's okay, and handled below. - files[l] = append(files[l], prevFile, f) - found = true - } - if f.Largest.IsExclusiveSentinel() { - prevUserKey = nil - prevFile = nil - } else { - prevUserKey = f.Largest.UserKey - prevFile = f - } - } - } - return - } -} - -// markFilesPrePebblev1 scans the LSM for files that do not support block -// properties (i.e. a table format version pre-Pebblev1). -var markFilesPrePebblev1 = func(tc *tableCacheContainer) findFilesFunc { - return func(v *version) (found bool, files [numLevels][]*fileMetadata, err error) { - for l := numLevels - 1; l > 0; l-- { - iter := v.Levels[l].Iter() - for f := iter.First(); f != nil; f = iter.Next() { - if f.Virtual { - // Any physical sstable which has been virtualized must - // have already undergone this migration, and we don't - // need to worry about the virtual sstable themselves. - panic("pebble: unexpected virtual sstable during migration") - } - err = tc.withReader( - f.PhysicalMeta(), func(r *sstable.Reader) error { - tf, err := r.TableFormat() - if err != nil { - return err - } - if tf < sstable.TableFormatPebblev1 { - found = true - files[l] = append(files[l], f) - } - return nil - }) - if err != nil { - return - } - } - } - return - } -} +// This method is not used currently, but it will be useful the next time we need +// to mark files for compaction. +var _ = (*DB)(nil).markFilesLocked -// markFilesLock durably marks the files that match the given findFilesFunc for +// markFilesLocked durably marks the files that match the given findFilesFunc for // compaction. func (d *DB) markFilesLocked(findFn findFilesFunc) error { jobID := d.mu.nextJobID diff --git a/format_major_version_test.go b/format_major_version_test.go index bbca42b5a49..8bf3d05b9f1 100644 --- a/format_major_version_test.go +++ b/format_major_version_test.go @@ -5,17 +5,9 @@ package pebble import ( - "bytes" "fmt" - "strconv" - "sync" "testing" - "time" - "github.com/cockroachdb/datadriven" - "github.com/cockroachdb/pebble/bloom" - "github.com/cockroachdb/pebble/internal/base" - "github.com/cockroachdb/pebble/internal/testkeys" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" "github.com/cockroachdb/pebble/vfs/atomicfs" @@ -23,7 +15,7 @@ import ( ) func TestFormatMajorVersion_MigrationDefined(t *testing.T) { - for v := FormatMostCompatible; v <= FormatNewest; v++ { + for v := FormatMinSupported; v <= FormatNewest; v++ { if _, ok := formatMajorVersionMigrations[v]; !ok { t.Errorf("format major version %d has no migration defined", v) } @@ -35,30 +27,6 @@ func TestRatchetFormat(t *testing.T) { d, err := Open("", (&Options{FS: fs}).WithFSDefaults()) require.NoError(t, err) require.NoError(t, d.Set([]byte("foo"), []byte("bar"), Sync)) - require.Equal(t, FormatMostCompatible, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatVersioned)) - require.Equal(t, FormatVersioned, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatVersioned)) - require.Equal(t, FormatVersioned, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatSetWithDelete)) - require.Equal(t, FormatSetWithDelete, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatBlockPropertyCollector)) - require.Equal(t, FormatBlockPropertyCollector, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatSplitUserKeysMarked)) - require.Equal(t, FormatSplitUserKeysMarked, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatSplitUserKeysMarkedCompacted)) - require.Equal(t, FormatSplitUserKeysMarkedCompacted, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatRangeKeys)) - require.Equal(t, FormatRangeKeys, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatMinTableFormatPebblev1)) - require.Equal(t, FormatMinTableFormatPebblev1, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1Marked)) - require.Equal(t, FormatPrePebblev1Marked, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(formatUnusedPrePebblev1MarkedCompacted)) - require.Equal(t, formatUnusedPrePebblev1MarkedCompacted, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatSSTableValueBlocks)) - require.Equal(t, FormatSSTableValueBlocks, d.FormatMajorVersion()) - require.NoError(t, d.RatchetFormatMajorVersion(FormatFlushableIngest)) require.Equal(t, FormatFlushableIngest, d.FormatMajorVersion()) require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1MarkedCompacted)) require.Equal(t, FormatPrePebblev1MarkedCompacted, d.FormatMajorVersion()) @@ -84,10 +52,10 @@ func TestRatchetFormat(t *testing.T) { _, err = Open("", (&Options{ FS: fs, - FormatMajorVersion: FormatVersioned, + FormatMajorVersion: FormatMinSupported, }).WithFSDefaults()) require.Error(t, err) - require.EqualError(t, err, `pebble: database "" written in format major version 999999`) + require.EqualError(t, err, `pebble: database "" written in unknown format major version 999999`) } func testBasicDB(d *DB) error { @@ -113,7 +81,7 @@ func testBasicDB(d *DB) error { } func TestFormatMajorVersions(t *testing.T) { - for vers := FormatMostCompatible; vers <= FormatNewest; vers++ { + for vers := FormatMinSupported; vers <= FormatNewest; vers++ { t.Run(fmt.Sprintf("vers=%03d", vers), func(t *testing.T) { fs := vfs.NewStrictMem() opts := (&Options{ @@ -214,27 +182,15 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) { // fixture is intentionally verbose. m := map[FormatMajorVersion][2]sstable.TableFormat{ - FormatDefault: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - FormatMostCompatible: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - formatVersionedManifestMarker: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - FormatVersioned: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - FormatSetWithDelete: {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2}, - FormatBlockPropertyCollector: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, - FormatSplitUserKeysMarked: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, - FormatSplitUserKeysMarkedCompacted: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1}, - FormatRangeKeys: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev2}, - FormatMinTableFormatPebblev1: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, - FormatPrePebblev1Marked: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, - formatUnusedPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2}, - FormatSSTableValueBlocks: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, - FormatFlushableIngest: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, - FormatPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, - FormatDeleteSizedAndObsolete: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, - FormatVirtualSSTables: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, + FormatDefault: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, + FormatFlushableIngest: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, + FormatPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3}, + FormatDeleteSizedAndObsolete: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, + FormatVirtualSSTables: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4}, } // Valid versions. - for fmv := FormatMostCompatible; fmv <= internalFormatNewest; fmv++ { + for fmv := FormatMinSupported; fmv <= internalFormatNewest; fmv++ { got := [2]sstable.TableFormat{fmv.MinTableFormat(), fmv.MaxTableFormat()} require.Equalf(t, m[fmv], got, "got %s; want %s", got, m[fmv]) require.True(t, got[0] <= got[1] /* min <= max */) @@ -245,336 +201,3 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) { require.Panics(t, func() { _ = fmv.MaxTableFormat() }) require.Panics(t, func() { _ = fmv.MinTableFormat() }) } - -func TestSplitUserKeyMigration(t *testing.T) { - var d *DB - var opts *Options - var fs vfs.FS - var buf bytes.Buffer - defer func() { - if d != nil { - require.NoError(t, d.Close()) - } - }() - - datadriven.RunTest(t, "testdata/format_major_version_split_user_key_migration", - func(t *testing.T, td *datadriven.TestData) string { - switch td.Cmd { - case "define": - if d != nil { - if err := d.Close(); err != nil { - return err.Error() - } - buf.Reset() - } - opts = (&Options{ - FormatMajorVersion: FormatBlockPropertyCollector, - EventListener: &EventListener{ - CompactionEnd: func(info CompactionInfo) { - // Fix the job ID and durations for determinism. - info.JobID = 100 - info.Duration = time.Second - info.TotalDuration = 2 * time.Second - fmt.Fprintln(&buf, info) - }, - }, - DisableAutomaticCompactions: true, - }).WithFSDefaults() - var err error - if d, err = runDBDefineCmd(td, opts); err != nil { - return err.Error() - } - - fs = d.opts.FS - d.mu.Lock() - defer d.mu.Unlock() - return d.mu.versions.currentVersion().DebugString(base.DefaultFormatter) - case "reopen": - if d != nil { - if err := d.Close(); err != nil { - return err.Error() - } - buf.Reset() - } - opts.FS = fs - opts.DisableAutomaticCompactions = true - var err error - d, err = Open("", opts) - if err != nil { - return err.Error() - } - return "OK" - case "build": - if err := runBuildCmd(td, d, fs); err != nil { - return err.Error() - } - return "" - case "force-ingest": - if err := runForceIngestCmd(td, d); err != nil { - return err.Error() - } - d.mu.Lock() - defer d.mu.Unlock() - return d.mu.versions.currentVersion().DebugString(base.DefaultFormatter) - case "format-major-version": - return d.FormatMajorVersion().String() - case "ratchet-format-major-version": - v, err := strconv.Atoi(td.CmdArgs[0].String()) - if err != nil { - return err.Error() - } - if err := d.RatchetFormatMajorVersion(FormatMajorVersion(v)); err != nil { - return err.Error() - } - return buf.String() - case "lsm": - return runLSMCmd(td, d) - case "marked-file-count": - m := d.Metrics() - return fmt.Sprintf("%d files marked for compaction", m.Compact.MarkedFiles) - case "disable-automatic-compactions": - d.mu.Lock() - defer d.mu.Unlock() - switch v := td.CmdArgs[0].String(); v { - case "true": - d.opts.DisableAutomaticCompactions = true - case "false": - d.opts.DisableAutomaticCompactions = false - default: - return fmt.Sprintf("unknown value %q", v) - } - return "" - default: - return fmt.Sprintf("unrecognized command %q", td.Cmd) - } - }) -} - -func TestPebblev1Migration(t *testing.T) { - var d *DB - defer func() { - if d != nil { - require.NoError(t, d.Close()) - } - }() - - datadriven.RunTest(t, "testdata/format_major_version_pebblev1_migration", - func(t *testing.T, td *datadriven.TestData) string { - switch cmd := td.Cmd; cmd { - case "open": - var version int - var err error - for _, cmdArg := range td.CmdArgs { - switch cmd := cmdArg.Key; cmd { - case "version": - version, err = strconv.Atoi(cmdArg.Vals[0]) - if err != nil { - return err.Error() - } - default: - return fmt.Sprintf("unknown argument: %s", cmd) - } - } - opts := (&Options{ - FS: vfs.NewMem(), - FormatMajorVersion: FormatMajorVersion(version), - }).WithFSDefaults() - d, err = Open("", opts) - if err != nil { - return err.Error() - } - return "" - - case "format-major-version": - return d.FormatMajorVersion().String() - - case "min-table-format": - return d.FormatMajorVersion().MinTableFormat().String() - - case "max-table-format": - return d.FormatMajorVersion().MaxTableFormat().String() - - case "disable-automatic-compactions": - d.mu.Lock() - defer d.mu.Unlock() - switch v := td.CmdArgs[0].String(); v { - case "true": - d.opts.DisableAutomaticCompactions = true - case "false": - d.opts.DisableAutomaticCompactions = false - default: - return fmt.Sprintf("unknown value %q", v) - } - return "" - - case "batch": - b := d.NewIndexedBatch() - if err := runBatchDefineCmd(td, b); err != nil { - return err.Error() - } - if err := b.Commit(nil); err != nil { - return err.Error() - } - return "" - - case "flush": - if err := d.Flush(); err != nil { - return err.Error() - } - return "" - - case "ingest": - if err := runBuildCmd(td, d, d.opts.FS); err != nil { - return err.Error() - } - // Only the first arg is a filename. - td.CmdArgs = td.CmdArgs[:1] - if err := runIngestCmd(td, d, d.opts.FS); err != nil { - return err.Error() - } - return "" - - case "lsm": - return runLSMCmd(td, d) - - case "tally-table-formats": - d.mu.Lock() - defer d.mu.Unlock() - v := d.mu.versions.currentVersion() - tally := make([]int, sstable.TableFormatMax+1) - for _, l := range v.Levels { - iter := l.Iter() - for m := iter.First(); m != nil; m = iter.Next() { - err := d.tableCache.withReader(m.PhysicalMeta(), - func(r *sstable.Reader) error { - f, err := r.TableFormat() - if err != nil { - return err - } - tally[f]++ - return nil - }) - if err != nil { - return err.Error() - } - } - } - var b bytes.Buffer - for i := 1; i <= int(sstable.TableFormatMax); i++ { - _, _ = fmt.Fprintf(&b, "%s: %d\n", sstable.TableFormat(i), tally[i]) - } - return b.String() - - case "ratchet-format-major-version": - v, err := strconv.Atoi(td.CmdArgs[0].String()) - if err != nil { - return err.Error() - } - if err = d.RatchetFormatMajorVersion(FormatMajorVersion(v)); err != nil { - return err.Error() - } - return "" - - case "marked-file-count": - m := d.Metrics() - return fmt.Sprintf("%d files marked for compaction", m.Compact.MarkedFiles) - - default: - return fmt.Sprintf("unknown command: %s", cmd) - } - }, - ) -} - -// TestPebblev1MigrationRace exercises the race between a PrePebbleV1Marked -// format major version upgrade that needs to open sstables to read their table -// format, and concurrent compactions that may delete the same files from the -// LSM. -// -// Regression test for #2019. -func TestPebblev1MigrationRace(t *testing.T) { - // Use a smaller table cache size to slow down the PrePebbleV1Marked - // migration, ensuring each table read needs to re-open the file. - cache := NewCache(4 << 20) - defer cache.Unref() - tableCache := NewTableCache(cache, 1, 5) - defer tableCache.Unref() - d, err := Open("", (&Options{ - Cache: cache, - FS: vfs.NewMem(), - FormatMajorVersion: FormatMajorVersion(FormatPrePebblev1Marked - 1), - TableCache: tableCache, - Levels: []LevelOptions{{TargetFileSize: 1}}, - }).WithFSDefaults()) - require.NoError(t, err) - defer d.Close() - - ks := testkeys.Alpha(3).EveryN(10) - var key [3]byte - for i := int64(0); i < ks.Count(); i++ { - n := testkeys.WriteKey(key[:], ks, i) - require.NoError(t, d.Set(key[:n], key[:n], nil)) - require.NoError(t, d.Flush()) - } - - // Asynchronously write and flush range deletes that will cause compactions - // to delete the existing sstables. These deletes will race with the format - // major version upgrade's migration will attempt to delete the files. - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for i := ks.Count() - 1; i > 0; i -= 50 { - endKey := testkeys.Key(ks, i) - startIndex := i - 50 - if startIndex < 0 { - startIndex = 0 - } - startKey := testkeys.Key(ks, startIndex) - - require.NoError(t, d.DeleteRange(startKey, endKey, nil)) - _, err := d.AsyncFlush() - require.NoError(t, err) - } - }() - require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1Marked)) - wg.Wait() -} - -// Regression test for #2044, where multiple concurrent compactions can lead -// to an indefinite wait on the compaction goroutine in compactMarkedFilesLocked. -func TestPebblev1MigrationConcurrencyRace(t *testing.T) { - opts := (&Options{ - Comparer: testkeys.Comparer, - FS: vfs.NewMem(), - FormatMajorVersion: FormatSplitUserKeysMarked, - Levels: []LevelOptions{{FilterPolicy: bloom.FilterPolicy(10)}}, - MaxConcurrentCompactions: func() int { - return 4 - }, - }).WithFSDefaults() - func() { - d, err := Open("", opts) - require.NoError(t, err) - defer func() { - require.NoError(t, d.Close()) - }() - - ks := testkeys.Alpha(3).EveryN(10) - var key [3]byte - for i := int64(0); i < ks.Count(); i++ { - n := testkeys.WriteKey(key[:], ks, i) - require.NoError(t, d.Set(key[:n], key[:n], nil)) - if i%100 == 0 { - require.NoError(t, d.Flush()) - } - } - require.NoError(t, d.Flush()) - }() - - opts.FormatMajorVersion = formatUnusedPrePebblev1MarkedCompacted - d, err := Open("", opts) - require.NoError(t, err) - require.NoError(t, d.RatchetFormatMajorVersion(formatUnusedPrePebblev1MarkedCompacted)) - require.NoError(t, d.Close()) -} diff --git a/get_iter_test.go b/get_iter_test.go index ab6e67e0a94..244c3378231 100644 --- a/get_iter_test.go +++ b/get_iter_test.go @@ -163,91 +163,6 @@ func TestGetIter(t *testing.T) { }, }, - { - description: "quad-4: four level-4 tables", - tables: []testTable{ - { - level: 4, - fileNum: 11, - data: []string{ - "aardvark.SET.101 a1", - "alpaca__.SET.201 a2", - "anteater.SET.301 a3", - }, - }, - { - level: 4, - fileNum: 22, - data: []string{ - "baboon__.SET.102 b1", - "baboon__.DEL.202 ", - "baboon__.SET.302 b3", - "bear____.SET.402 b4", - "bear____.DEL.502 ", - "buffalo_.SET.602 b6", - }, - }, - { - level: 4, - fileNum: 33, - data: []string{ - "buffalo_.SET.103 B1", - }, - }, - { - level: 4, - fileNum: 44, - data: []string{ - "chipmunk.SET.104 c1", - "chipmunk.SET.204 c2", - }, - }, - }, - queries: []string{ - "a_______.SEPARATOR.999 ErrNotFound", - "aardvark.SEPARATOR.999 a1", - "aardvark.SEPARATOR.102 a1", - "aardvark.SEPARATOR.101 a1", - "aardvark.SEPARATOR.100 ErrNotFound", - "alpaca__.SEPARATOR.999 a2", - "alpaca__.SEPARATOR.200 ErrNotFound", - "anteater.SEPARATOR.999 a3", - "anteater.SEPARATOR.302 a3", - "anteater.SEPARATOR.301 a3", - "anteater.SEPARATOR.300 ErrNotFound", - "anteater.SEPARATOR.000 ErrNotFound", - "b_______.SEPARATOR.999 ErrNotFound", - "baboon__.SEPARATOR.999 b3", - "baboon__.SEPARATOR.302 b3", - "baboon__.SEPARATOR.301 ErrNotFound", - "baboon__.SEPARATOR.202 ErrNotFound", - "baboon__.SEPARATOR.201 b1", - "baboon__.SEPARATOR.102 b1", - "baboon__.SEPARATOR.101 ErrNotFound", - "bear____.SEPARATOR.999 ErrNotFound", - "bear____.SEPARATOR.500 b4", - "bear____.SEPARATOR.000 ErrNotFound", - "buffalo_.SEPARATOR.999 b6", - "buffalo_.SEPARATOR.603 b6", - "buffalo_.SEPARATOR.602 b6", - "buffalo_.SEPARATOR.601 B1", - "buffalo_.SEPARATOR.104 B1", - "buffalo_.SEPARATOR.103 B1", - "buffalo_.SEPARATOR.102 ErrNotFound", - "buffalo_.SEPARATOR.000 ErrNotFound", - "c_______.SEPARATOR.999 ErrNotFound", - "chipmunk.SEPARATOR.999 c2", - "chipmunk.SEPARATOR.205 c2", - "chipmunk.SEPARATOR.204 c2", - "chipmunk.SEPARATOR.203 c1", - "chipmunk.SEPARATOR.105 c1", - "chipmunk.SEPARATOR.104 c1", - "chipmunk.SEPARATOR.103 ErrNotFound", - "chipmunk.SEPARATOR.000 ErrNotFound", - "d_______.SEPARATOR.999 ErrNotFound", - }, - }, - { description: "complex: many tables at many levels", tables: []testTable{ @@ -515,7 +430,7 @@ func TestGetIter(t *testing.T) { files[tt.level] = append(files[tt.level], meta) } v := manifest.NewVersion(cmp, base.DefaultFormatter, 10<<20, files) - err := v.CheckOrdering(cmp, base.DefaultFormatter, manifest.AllowSplitUserKeys) + err := v.CheckOrdering(cmp, base.DefaultFormatter) if tc.badOrdering && err == nil { t.Errorf("desc=%q: want bad ordering, got nil error", desc) continue diff --git a/ingest_test.go b/ingest_test.go index c4dcc2aa314..581803d96a3 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -2493,54 +2493,47 @@ func TestConcurrentIngestCompact(t *testing.T) { func TestIngestFlushQueuedMemTable(t *testing.T) { // Verify that ingestion forces a flush of a queued memtable. - // Test with a format major version prior to FormatFlushableIngest and one - // after. Both should result in the same statistic calculations. - for _, fmv := range []FormatMajorVersion{FormatFlushableIngest - 1, internalFormatNewest} { - func(fmv FormatMajorVersion) { - mem := vfs.NewMem() - d, err := Open("", &Options{ - FS: mem, - FormatMajorVersion: fmv, - }) - require.NoError(t, err) + mem := vfs.NewMem() + o := &Options{FS: mem} + o.testingRandomized(t) + d, err := Open("", o) + require.NoError(t, err) - // Add the key "a" to the memtable, then fill up the memtable with the key - // "b". The ingested sstable will only overlap with the queued memtable. - require.NoError(t, d.Set([]byte("a"), nil, nil)) - for { - require.NoError(t, d.Set([]byte("b"), nil, nil)) - d.mu.Lock() - done := len(d.mu.mem.queue) == 2 - d.mu.Unlock() - if done { - break - } - } + // Add the key "a" to the memtable, then fill up the memtable with the key + // "b". The ingested sstable will only overlap with the queued memtable. + require.NoError(t, d.Set([]byte("a"), nil, nil)) + for { + require.NoError(t, d.Set([]byte("b"), nil, nil)) + d.mu.Lock() + done := len(d.mu.mem.queue) == 2 + d.mu.Unlock() + if done { + break + } + } - ingest := func(keys ...string) { - t.Helper() - f, err := mem.Create("ext") - require.NoError(t, err) + ingest := func(keys ...string) { + t.Helper() + f, err := mem.Create("ext") + require.NoError(t, err) - w := sstable.NewWriter(objstorageprovider.NewFileWritable(f), sstable.WriterOptions{ - TableFormat: fmv.MinTableFormat(), - }) - for _, k := range keys { - require.NoError(t, w.Set([]byte(k), nil)) - } - require.NoError(t, w.Close()) - stats, err := d.IngestWithStats([]string{"ext"}) - require.NoError(t, err) - require.Equal(t, stats.ApproxIngestedIntoL0Bytes, stats.Bytes) - require.Equal(t, stats.MemtableOverlappingFiles, 1) - require.Less(t, uint64(0), stats.Bytes) - } + w := sstable.NewWriter(objstorageprovider.NewFileWritable(f), sstable.WriterOptions{ + TableFormat: o.FormatMajorVersion.MinTableFormat(), + }) + for _, k := range keys { + require.NoError(t, w.Set([]byte(k), nil)) + } + require.NoError(t, w.Close()) + stats, err := d.IngestWithStats([]string{"ext"}) + require.NoError(t, err) + require.Equal(t, stats.ApproxIngestedIntoL0Bytes, stats.Bytes) + require.Equal(t, stats.MemtableOverlappingFiles, 1) + require.Less(t, uint64(0), stats.Bytes) + } - ingest("a") + ingest("a") - require.NoError(t, d.Close()) - }(fmv) - } + require.NoError(t, d.Close()) } func TestIngestStats(t *testing.T) { diff --git a/internal/base/filenames.go b/internal/base/filenames.go index 06098ab6396..c4420ffb76c 100644 --- a/internal/base/filenames.go +++ b/internal/base/filenames.go @@ -62,7 +62,6 @@ const ( FileTypeLock FileTypeTable FileTypeManifest - FileTypeCurrent FileTypeOptions FileTypeOldTemp FileTypeTemp @@ -79,8 +78,6 @@ func MakeFilename(fileType FileType, dfn DiskFileNum) string { return fmt.Sprintf("%s.sst", dfn) case FileTypeManifest: return fmt.Sprintf("MANIFEST-%s", dfn) - case FileTypeCurrent: - return "CURRENT" case FileTypeOptions: return fmt.Sprintf("OPTIONS-%s", dfn) case FileTypeOldTemp: @@ -100,8 +97,6 @@ func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) { filename = fs.PathBase(filename) switch { - case filename == "CURRENT": - return FileTypeCurrent, 0, true case filename == "LOCK": return FileTypeLock, 0, true case strings.HasPrefix(filename, "MANIFEST-"): diff --git a/internal/base/filenames_test.go b/internal/base/filenames_test.go index 9a5c331580e..6a52b759ebf 100644 --- a/internal/base/filenames_test.go +++ b/internal/base/filenames_test.go @@ -55,9 +55,8 @@ func TestParseFilename(t *testing.T) { func TestFilenameRoundTrip(t *testing.T) { testCases := map[FileType]bool{ - // CURRENT and LOCK files aren't numbered. - FileTypeCurrent: false, - FileTypeLock: false, + // LOCK files aren't numbered. + FileTypeLock: false, // The remaining file types are numbered. FileTypeLog: true, FileTypeManifest: true, diff --git a/internal/keyspan/level_iter_test.go b/internal/keyspan/level_iter_test.go index 6e3039677a5..d9897dc28b7 100644 --- a/internal/keyspan/level_iter_test.go +++ b/internal/keyspan/level_iter_test.go @@ -309,7 +309,7 @@ func TestLevelIterEquivalence(t *testing.T) { amap[metas[i].FileNum] = metas[i] } b.Added[6] = amap - v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil, manifest.ProhibitSplitUserKeys) + v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil) require.NoError(t, err) levelIter.Init( SpanIterOptions{}, base.DefaultComparer.Compare, tableNewIters, @@ -448,7 +448,7 @@ func TestLevelIter(t *testing.T) { amap[metas[i].FileNum] = metas[i] } b.Added[6] = amap - v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil, manifest.ProhibitSplitUserKeys) + v, err := b.Apply(nil, base.DefaultComparer.Compare, base.DefaultFormatter, 0, 0, nil) require.NoError(t, err) iter = NewLevelIter( SpanIterOptions{}, base.DefaultComparer.Compare, diff --git a/internal/manifest/l0_sublevels_test.go b/internal/manifest/l0_sublevels_test.go index 8cedb87358b..2b007fca9e7 100644 --- a/internal/manifest/l0_sublevels_test.go +++ b/internal/manifest/l0_sublevels_test.go @@ -51,7 +51,7 @@ func readManifest(filename string) (*Version, error) { if err := bve.Accumulate(&ve); err != nil { return nil, err } - if v, err = bve.Apply(v, base.DefaultComparer.Compare, base.DefaultFormatter, 10<<20, 32000, nil, ProhibitSplitUserKeys); err != nil { + if v, err = bve.Apply(v, base.DefaultComparer.Compare, base.DefaultFormatter, 10<<20, 32000, nil); err != nil { return nil, err } } @@ -448,7 +448,7 @@ func TestL0Sublevels(t *testing.T) { for sublevel, files := range sublevels.levelFiles { slice := NewLevelSliceSpecificOrder(files) err := CheckOrdering(base.DefaultComparer.Compare, base.DefaultFormatter, - L0Sublevel(sublevel), slice.Iter(), ProhibitSplitUserKeys) + L0Sublevel(sublevel), slice.Iter()) if err != nil { return err.Error() } diff --git a/internal/manifest/testdata/version_check_ordering b/internal/manifest/testdata/version_check_ordering index 9f8f710056a..58fd4cef22a 100644 --- a/internal/manifest/testdata/version_check_ordering +++ b/internal/manifest/testdata/version_check_ordering @@ -179,13 +179,6 @@ L1 files 000001 and 000002 have overlapping ranges: [a#1,SET-b#2,SET] vs [b#1,SE 000001:[a#1,SET-b#2,SET] seqnums:[0-0] points:[a#1,SET-b#2,SET] 000002:[b#1,SET-d#4,SET] seqnums:[0-0] points:[b#1,SET-d#4,SET] -check-ordering allow-split-user-keys -1: - 000001:[a#1,SET-b#2,SET] - 000002:[b#1,SET-d#4,SET] ----- -OK - check-ordering 1: 000001:[a#1,SET-b#2,SET] diff --git a/internal/manifest/version.go b/internal/manifest/version.go index d20240af976..2b874331c1b 100644 --- a/internal/manifest/version.go +++ b/internal/manifest/version.go @@ -1343,20 +1343,16 @@ func (v *Version) Overlaps( // CheckOrdering checks that the files are consistent with respect to // increasing file numbers (for level 0 files) and increasing and non- // overlapping internal key ranges (for level non-0 files). -func (v *Version) CheckOrdering( - cmp Compare, format base.FormatKey, order OrderingInvariants, -) error { +func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error { for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- { sublevelIter := v.L0SublevelFiles[sublevel].Iter() - // Sublevels have NEVER allowed split user keys, so we can pass - // ProhibitSplitUserKeys. - if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter, ProhibitSplitUserKeys); err != nil { + if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), sublevelIter); err != nil { return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format)) } } for level, lm := range v.Levels { - if err := CheckOrdering(cmp, format, Level(level), lm.Iter(), order); err != nil { + if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil { return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format)) } } @@ -1425,34 +1421,10 @@ func (l *VersionList) Remove(v *Version) { v.list = nil // avoid memory leaks } -// OrderingInvariants dictates the file ordering invariants active. -type OrderingInvariants int8 - -const ( - // ProhibitSplitUserKeys indicates that adjacent files within a level cannot - // contain the same user key. - ProhibitSplitUserKeys OrderingInvariants = iota - // AllowSplitUserKeys indicates that adjacent files within a level may - // contain the same user key. This is only allowed by historical format - // major versions. - // - // TODO(jackson): Remove. - AllowSplitUserKeys -) - // CheckOrdering checks that the files are consistent with respect to // seqnums (for level 0 files -- see detailed comment below) and increasing and non- // overlapping internal key ranges (for non-level 0 files). -// -// The ordering field may be passed AllowSplitUserKeys to allow adjacent files that are both -// inclusive of the same user key. Pebble no longer creates version edits -// installing such files, and Pebble databases with sufficiently high format -// major version should no longer have any such files within their LSM. -// TODO(jackson): Remove AllowSplitUserKeys when we remove support for the -// earlier format major versions. -func CheckOrdering( - cmp Compare, format base.FormatKey, level Level, files LevelIterator, ordering OrderingInvariants, -) error { +func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error { // The invariants to check for L0 sublevels are the same as the ones to // check for all other levels. However, if L0 is not organized into // sublevels, or if all L0 files are being passed in, we do the legacy L0 @@ -1531,28 +1503,15 @@ func CheckOrdering( f.Smallest.Pretty(format), f.Largest.Pretty(format)) } - // What's considered "overlapping" is dependent on the format - // major version. If ordering=ProhibitSplitUserKeys, then both - // files cannot contain keys with the same user keys. If the - // bounds have the same user key, the previous file's boundary - // must have a Trailer indicating that it's exclusive. - switch ordering { - case AllowSplitUserKeys: - if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 { - return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", - errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), - prev.Smallest.Pretty(format), prev.Largest.Pretty(format), - f.Smallest.Pretty(format), f.Largest.Pretty(format)) - } - case ProhibitSplitUserKeys: - if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) { - return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", - errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), - prev.Smallest.Pretty(format), prev.Largest.Pretty(format), - f.Smallest.Pretty(format), f.Largest.Pretty(format)) - } - default: - panic("unreachable") + // In all supported format major version, split user keys are + // prohibited, so both files cannot contain keys with the same user + // keys. If the bounds have the same user key, the previous file's + // boundary must have a Trailer indicating that it's exclusive. + if v := cmp(prev.Largest.UserKey, f.Smallest.UserKey); v > 0 || (v == 0 && !prev.Largest.IsExclusiveSentinel()) { + return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", + errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), + prev.Smallest.Pretty(format), prev.Largest.Pretty(format), + f.Smallest.Pretty(format), f.Largest.Pretty(format)) } } } diff --git a/internal/manifest/version_edit.go b/internal/manifest/version_edit.go index 08b3555e355..956a56451b0 100644 --- a/internal/manifest/version_edit.go +++ b/internal/manifest/version_edit.go @@ -855,7 +855,6 @@ func AccumulateIncompleteAndApplySingleVE( backingStateMap map[base.DiskFileNum]*FileBacking, addBackingFunc func(*FileBacking), removeBackingFunc func(base.DiskFileNum), - orderingInvariants OrderingInvariants, ) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) { if len(ve.RemovedBackingTables) != 0 { panic("pebble: invalid incomplete version edit") @@ -866,9 +865,7 @@ func AccumulateIncompleteAndApplySingleVE( return nil, nil, err } zombies = make(map[base.DiskFileNum]uint64) - v, err := b.Apply( - curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, orderingInvariants, - ) + v, err := b.Apply(curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies) if err != nil { return nil, nil, err } @@ -908,7 +905,6 @@ func (b *BulkVersionEdit) Apply( flushSplitBytes int64, readCompactionRate int64, zombies map[base.DiskFileNum]uint64, - orderingInvariants OrderingInvariants, ) (*Version, error) { addZombie := func(state *FileBacking) { if zombies != nil { @@ -1092,7 +1088,7 @@ func (b *BulkVersionEdit) Apply( } else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { return nil, errors.Wrap(err, "pebble: internal error") } - if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter(), orderingInvariants); err != nil { + if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil { return nil, errors.Wrap(err, "pebble: internal error") } continue @@ -1113,7 +1109,7 @@ func (b *BulkVersionEdit) Apply( end.Prev() } }) - if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter(), orderingInvariants); err != nil { + if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil { return nil, errors.Wrap(err, "pebble: internal error") } } diff --git a/internal/manifest/version_edit_test.go b/internal/manifest/version_edit_test.go index 9ad02c55c57..3ae79922506 100644 --- a/internal/manifest/version_edit_test.go +++ b/internal/manifest/version_edit_test.go @@ -514,7 +514,7 @@ func TestVersionEditApply(t *testing.T) { } } zombies := make(map[base.DiskFileNum]uint64) - newv, err := bve.Apply(v, base.DefaultComparer.Compare, base.DefaultFormatter, 10<<20, 32000, zombies, ProhibitSplitUserKeys) + newv, err := bve.Apply(v, base.DefaultComparer.Compare, base.DefaultFormatter, 10<<20, 32000, zombies) if err != nil { return err.Error() } diff --git a/internal/manifest/version_test.go b/internal/manifest/version_test.go index abde613afc5..eb743adad8f 100644 --- a/internal/manifest/version_test.go +++ b/internal/manifest/version_test.go @@ -290,10 +290,6 @@ func TestCheckOrdering(t *testing.T) { func(t *testing.T, d *datadriven.TestData) string { switch d.Cmd { case "check-ordering": - orderingInvariants := ProhibitSplitUserKeys - if d.HasArg("allow-split-user-keys") { - orderingInvariants = AllowSplitUserKeys - } v, err := ParseVersionDebug(cmp, fmtKey, 10<<20, d.Input) if err != nil { return err.Error() @@ -304,7 +300,7 @@ func TestCheckOrdering(t *testing.T) { m.SmallestSeqNum = m.Smallest.SeqNum() m.LargestSeqNum = m.Largest.SeqNum() }) - if err = v.CheckOrdering(cmp, base.DefaultFormatter, orderingInvariants); err != nil { + if err = v.CheckOrdering(cmp, base.DefaultFormatter); err != nil { return err.Error() } return "OK" diff --git a/iterator_histories_test.go b/iterator_histories_test.go index 792b2faf2fc..b6d40e0ac07 100644 --- a/iterator_histories_test.go +++ b/iterator_histories_test.go @@ -45,7 +45,7 @@ func TestIterHistories(t *testing.T) { opts = &Options{ FS: vfs.NewMem(), Comparer: testkeys.Comparer, - FormatMajorVersion: FormatRangeKeys, + FormatMajorVersion: FormatMinSupported, BlockPropertyCollectors: []func() BlockPropertyCollector{ sstable.NewTestKeysBlockPropertyCollector, }, diff --git a/iterator_test.go b/iterator_test.go index 0be563dc715..db8f6c191f1 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -1751,11 +1751,11 @@ func iterOptionsString(o *IterOptions) string { func newTestkeysDatabase(t *testing.T, ks testkeys.Keyspace, rng *rand.Rand) *DB { dbOpts := &Options{ - Comparer: testkeys.Comparer, - FS: vfs.NewMem(), - FormatMajorVersion: FormatRangeKeys, - Logger: panicLogger{}, + Comparer: testkeys.Comparer, + FS: vfs.NewMem(), + Logger: panicLogger{}, } + dbOpts.testingRandomized(t) d, err := Open("", dbOpts) require.NoError(t, err) @@ -1804,8 +1804,8 @@ func newPointTestkeysDatabase(t *testing.T, ks testkeys.Keyspace) *DB { dbOpts := &Options{ Comparer: testkeys.Comparer, FS: vfs.NewMem(), - FormatMajorVersion: FormatRangeKeys, } + dbOpts.testingRandomized(t) d, err := Open("", dbOpts) require.NoError(t, err) diff --git a/level_iter_test.go b/level_iter_test.go index 63d448e68bc..eba201517ad 100644 --- a/level_iter_test.go +++ b/level_iter_test.go @@ -193,12 +193,10 @@ func (lt *levelIterTest) runBuild(d *datadriven.TestData) string { return err.Error() } - tableFormat := sstable.TableFormatRocksDBv2 + tableFormat := sstable.TableFormatMinSupported for _, arg := range d.CmdArgs { if arg.Key == "format" { switch arg.Vals[0] { - case "rocksdbv2": - tableFormat = sstable.TableFormatRocksDBv2 case "pebblev2": tableFormat = sstable.TableFormatPebblev2 } diff --git a/metamorphic/options.go b/metamorphic/options.go index 394d7daf942..36f41d6e7da 100644 --- a/metamorphic/options.go +++ b/metamorphic/options.go @@ -26,16 +26,13 @@ import ( ) const ( - // The metamorphic test exercises range keys, so we cannot use an older - // FormatMajorVersion than pebble.FormatRangeKeys. - minimumFormatMajorVersion = pebble.FormatRangeKeys + minimumFormatMajorVersion = pebble.FormatMinSupported // The format major version to use in the default options configurations. We - // default to the last format major version of Cockroach 22.2 so we exercise - // the runtime version ratcheting that a cluster upgrading to 23.1 would - // experience. The randomized options may still use format major versions - // that are less than defaultFormatMajorVersion but are at least - // minimumFormatMajorVersion. - defaultFormatMajorVersion = pebble.FormatPrePebblev1Marked + // default to the minimum supported format so we exercise the runtime version + // ratcheting that a cluster upgrading would experience. The randomized + // options may still use format major versions that are less than + // defaultFormatMajorVersion but are at least minimumFormatMajorVersion. + defaultFormatMajorVersion = pebble.FormatMinSupported // newestFormatMajorVersionToTest is the most recent format major version // the metamorphic tests should use. This may be greater than // pebble.FormatNewest when some format major versions are marked as diff --git a/open.go b/open.go index d0b5e189bd8..fff1d2cf6d3 100644 --- a/open.go +++ b/open.go @@ -71,7 +71,7 @@ func TableCacheSize(maxOpenFiles int) int { } // Open opens a DB whose files live in the given directory. -func Open(dirname string, opts *Options) (db *DB, _ error) { +func Open(dirname string, opts *Options) (db *DB, err error) { // Make a copy of the options so that we don't mutate the passed in options. opts = opts.Clone() opts = opts.EnsureDefaults() @@ -136,8 +136,28 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } }() + noFormatVersionMarker := formatVersion == FormatDefault + if noFormatVersionMarker { + // There is no format version marker file. There are three cases: + // - we are trying to open an existing store that was created at + // FormatMostCompatible (the only one without a version marker file) + // - we are creating a new store; + // - we are retrying a failed creation. + // + // To error in the first case, we set ErrorIfPristine. + opts.ErrorIfNotPristine = true + formatVersion = FormatMinSupported + defer func() { + if err != nil && errors.Is(err, ErrDBNotPristine) { + // We must be trying to open an existing store at FormatMostCompatible. + // Correct the error in this case -we + err = errors.Newf("pebble: database %q written in format major version 1 which is no longer supported", dirname) + } + }() + } + // Find the currently active manifest, if there is one. - manifestMarker, manifestFileNum, manifestExists, err := findCurrentManifest(formatVersion, opts.FS, dirname) + manifestMarker, manifestFileNum, manifestExists, err := findCurrentManifest(opts.FS, dirname) if err != nil { return nil, errors.Wrapf(err, "pebble: database %q", dirname) } @@ -252,8 +272,6 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { jobID := d.mu.nextJobID d.mu.nextJobID++ - setCurrent := setCurrentFunc(d.FormatMajorVersion(), manifestMarker, opts.FS, dirname, d.dataDir) - if !manifestExists { // DB does not exist. if d.opts.ErrorIfNotExists || d.opts.ReadOnly { @@ -261,7 +279,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } // Create the DB. - if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil { + if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, d.FormatMajorVersion, &d.mu.Mutex); err != nil { return nil, err } } else { @@ -269,7 +287,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { return nil, errors.Wrapf(ErrDBAlreadyExists, "dirname=%q", dirname) } // Load the version set. - if err := d.mu.versions.load(dirname, opts, manifestFileNum, manifestMarker, setCurrent, d.FormatMajorVersion, &d.mu.Mutex); err != nil { + if err := d.mu.versions.load(dirname, opts, manifestFileNum, manifestMarker, d.FormatMajorVersion, &d.mu.Mutex); err != nil { return nil, err } if opts.ErrorIfNotPristine { @@ -485,20 +503,27 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } d.updateReadStateLocked(d.opts.DebugCheck) - // If the Options specify a format major version higher than the - // loaded database's, upgrade it. If this is a new database, this - // code path also performs an initial upgrade from the starting - // implicit MostCompatible version. - // - // We ratchet the version this far into Open so that migrations have a read - // state available. - if !d.opts.ReadOnly && opts.FormatMajorVersion > d.FormatMajorVersion() { - if err := d.ratchetFormatMajorVersionLocked(opts.FormatMajorVersion); err != nil { - return nil, err + if !d.opts.ReadOnly { + // If the Options specify a format major version higher than the + // loaded database's, upgrade it. If this is a new database, this + // code path also performs an initial upgrade from the starting + // implicit MinSupported version. + // + // We ratchet the version this far into Open so that migrations have a read + // state available. Note that this also results in creating/updating the + // format version marker file. + if opts.FormatMajorVersion > d.FormatMajorVersion() { + if err := d.ratchetFormatMajorVersionLocked(opts.FormatMajorVersion); err != nil { + return nil, err + } + } else if noFormatVersionMarker { + // We are creating a new store at MinSupported. Create the format version + // marker file. + if err := d.writeFormatVersionMarker(d.FormatMajorVersion()); err != nil { + return nil, err + } } - } - if !d.opts.ReadOnly { // Write the current options to disk. d.optionsFileNum = d.mu.versions.getNextDiskFileNum() tmpPath := base.MakeFilepath(opts.FS, dirname, fileTypeTemp, d.optionsFileNum) @@ -994,7 +1019,7 @@ func (d *DB) replayWAL( flushMem() // mem is nil here. - if !d.opts.ReadOnly { + if !d.opts.ReadOnly && batchesReplayed > 0 { err = updateVE() if err != nil { return nil, 0, err @@ -1044,7 +1069,7 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) { } // Find the currently active manifest, if there is one. - manifestMarker, manifestFileNum, exists, err := findCurrentManifest(vers, fs, dirname) + manifestMarker, manifestFileNum, exists, err := findCurrentManifest(fs, dirname) if err != nil { return nil, err } diff --git a/open_test.go b/open_test.go index fae3237aaca..9749068d22b 100644 --- a/open_test.go +++ b/open_test.go @@ -187,20 +187,12 @@ func TestOpenAlreadyLocked(t *testing.T) { func TestNewDBFilenames(t *testing.T) { versions := map[FormatMajorVersion][]string{ - FormatMostCompatible: { - "000002.log", - "CURRENT", - "LOCK", - "MANIFEST-000001", - "OPTIONS-000003", - }, internalFormatNewest: { "000002.log", - "CURRENT", "LOCK", "MANIFEST-000001", "OPTIONS-000003", - "marker.format-version.000015.016", + "marker.format-version.000003.016", "marker.manifest.000001.MANIFEST-000001", }, } @@ -1115,42 +1107,11 @@ func TestGetVersion(t *testing.T) { require.Equal(t, "rocksdb v6.2.1", version) } -func TestRocksDBNoFlushManifest(t *testing.T) { - mem := vfs.NewMem() - // Have the comparer and merger names match what's in the testdata - // directory. - comparer := *DefaultComparer - merger := *DefaultMerger - comparer.Name = "cockroach_comparator" - merger.Name = "cockroach_merge_operator" - opts := &Options{ - FS: mem, - Comparer: &comparer, - Merger: &merger, - } - - // rocksdb-ingest-only is a RocksDB-generated db directory that has not had - // a single flush yet, only ingestion operations. The manifest contains - // a next-log-num but no log-num entry. Ensure that pebble can read these - // directories without an issue. - _, err := vfs.Clone(vfs.Default, mem, "testdata/rocksdb-ingest-only", "testdata") - require.NoError(t, err) - - db, err := Open("testdata", opts) - require.NoError(t, err) - defer db.Close() - - val, closer, err := db.Get([]byte("ajulxeiombjiyw\x00\x00\x00\x00\x00\x00\x00\x01\x12\x09")) - require.NoError(t, err) - require.NotEmpty(t, val) - require.NoError(t, closer.Close()) -} - func TestOpen_ErrorIfUnknownFormatVersion(t *testing.T) { fs := vfs.NewMem() d, err := Open("", &Options{ FS: fs, - FormatMajorVersion: FormatVersioned, + FormatMajorVersion: FormatMinSupported, }) require.NoError(t, err) require.NoError(t, d.Close()) @@ -1163,10 +1124,10 @@ func TestOpen_ErrorIfUnknownFormatVersion(t *testing.T) { _, err = Open("", &Options{ FS: fs, - FormatMajorVersion: FormatVersioned, + FormatMajorVersion: FormatMinSupported, }) require.Error(t, err) - require.EqualError(t, err, `pebble: database "" written in format major version 999999`) + require.EqualError(t, err, `pebble: database "" written in unknown format major version 999999`) } // ensureFilesClosed updates the provided Options to wrap the filesystem. It diff --git a/options.go b/options.go index 244b101249c..2beee1fa88e 100644 --- a/options.go +++ b/options.go @@ -1097,7 +1097,7 @@ func (o *Options) EnsureDefaults() *Options { } if o.FormatMajorVersion == FormatDefault { - o.FormatMajorVersion = FormatMostCompatible + o.FormatMajorVersion = FormatMinSupported } if o.FS == nil { @@ -1461,7 +1461,7 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { var v uint64 v, err = strconv.ParseUint(value, 10, 64) if vers := FormatMajorVersion(v); vers > internalFormatNewest || vers == FormatDefault { - err = errors.Newf("unknown format major version %d", o.FormatMajorVersion) + err = errors.Newf("unsupported format major version %d", o.FormatMajorVersion) } if err == nil { o.FormatMajorVersion = FormatMajorVersion(v) @@ -1686,9 +1686,9 @@ func (o *Options) Validate() error { fmt.Fprintf(&buf, "MemTableStopWritesThreshold (%d) must be >= 2\n", o.MemTableStopWritesThreshold) } - if o.FormatMajorVersion > internalFormatNewest { - fmt.Fprintf(&buf, "FormatMajorVersion (%d) must be <= %d\n", - o.FormatMajorVersion, internalFormatNewest) + if o.FormatMajorVersion < FormatMinSupported || o.FormatMajorVersion > internalFormatNewest { + fmt.Fprintf(&buf, "FormatMajorVersion (%d) must be between %d and %d\n", + o.FormatMajorVersion, FormatMinSupported, internalFormatNewest) } if o.TableCache != nil && o.Cache != o.TableCache.cache { fmt.Fprintf(&buf, "underlying cache in the TableCache and the Cache dont match\n") diff --git a/options_test.go b/options_test.go index 46a5863ee6f..58fc8e4aeac 100644 --- a/options_test.go +++ b/options_test.go @@ -28,8 +28,9 @@ func (o *Options) testingRandomized(t testing.TB) *Options { } if o.FormatMajorVersion == FormatDefault { // Pick a random format major version from the range - // [MostCompatible, FormatNewest]. - o.FormatMajorVersion = FormatMajorVersion(rand.Intn(int(internalFormatNewest)) + 1) + // [FormatMinSupported, FormatNewest]. + n := rand.Intn(int(internalFormatNewest - FormatMinSupported + 1)) + o.FormatMajorVersion = FormatMinSupported + FormatMajorVersion(n) t.Logf("Running %s with format major version %s", t.Name(), o.FormatMajorVersion.String()) } return o @@ -82,7 +83,7 @@ func TestOptionsString(t *testing.T) { flush_delay_delete_range=0s flush_delay_range_key=0s flush_split_bytes=4194304 - format_major_version=1 + format_major_version=13 l0_compaction_concurrency=10 l0_compaction_file_threshold=500 l0_compaction_threshold=4 @@ -121,9 +122,7 @@ func TestOptionsString(t *testing.T) { var opts *Options opts = opts.EnsureDefaults() - if v := opts.String(); expected != v { - t.Fatalf("expected\n%s\nbut found\n%s", expected, v) - } + require.Equal(t, expected, opts.String()) } func TestOptionsCheck(t *testing.T) { diff --git a/range_del_test.go b/range_del_test.go index 75a9ad2ee68..07251ab33a4 100644 --- a/range_del_test.go +++ b/range_del_test.go @@ -352,17 +352,7 @@ func TestRangeDelCompactionTruncation(t *testing.T) { // tables in L2. Lastly, the L2 table containing "c" will be compacted // creating the L3 table. require.NoError(t, d.Compact([]byte("c"), []byte("c\x00"), false)) - if formatVersion < FormatSetWithDelete { - expectLSM(` -1: - 000008:[a#12,RANGEDEL-b#inf,RANGEDEL] -2: - 000012:[b#13,SET-c#inf,RANGEDEL] -3: - 000013:[c#14,SET-d#inf,RANGEDEL] -`) - } else { - expectLSM(` + expectLSM(` 1: 000008:[a#12,RANGEDEL-b#inf,RANGEDEL] 2: @@ -370,7 +360,6 @@ func TestRangeDelCompactionTruncation(t *testing.T) { 3: 000013:[c#14,SET-d#inf,RANGEDEL] `) - } // The L1 table still contains a tombstone from [a,d) which will improperly // delete the newer version of "b" in L2. @@ -386,9 +375,7 @@ func TestRangeDelCompactionTruncation(t *testing.T) { } versions := []FormatMajorVersion{ - FormatMostCompatible, - FormatSetWithDelete - 1, - FormatSetWithDelete, + FormatMinSupported, FormatNewest, } for _, version := range versions { diff --git a/replay/replay.go b/replay/replay.go index d1d894f2ff3..106c8f0534f 100644 --- a/replay/replay.go +++ b/replay/replay.go @@ -718,8 +718,7 @@ func (r *Runner) prepareWorkloadSteps(ctx context.Context) error { r.Opts.Comparer.FormatKey, r.Opts.FlushSplitBytes, r.Opts.Experimental.ReadCompactionRate, - nil, /* zombies */ - manifest.ProhibitSplitUserKeys) + nil /* zombies */) bve = manifest.BulkVersionEdit{AddedByFileNum: bve.AddedByFileNum} return v, err } diff --git a/replay/replay_test.go b/replay/replay_test.go index 93bbdbef4ba..f924edecef8 100644 --- a/replay/replay_test.go +++ b/replay/replay_test.go @@ -91,7 +91,7 @@ func runReplayTest(t *testing.T, path string) { opts := &pebble.Options{ FS: fs, Comparer: testkeys.Comparer, - FormatMajorVersion: pebble.FormatRangeKeys, + FormatMajorVersion: pebble.FormatMinSupported, L0CompactionFileThreshold: 1, } setDefaultExperimentalOpts(opts) @@ -174,7 +174,7 @@ func TestLoadFlushedSSTableKeys(t *testing.T) { }, FS: vfs.NewMem(), Comparer: testkeys.Comparer, - FormatMajorVersion: pebble.FormatRangeKeys, + FormatMajorVersion: pebble.FormatMinSupported, } d, err := pebble.Open("", opts) require.NoError(t, err) @@ -282,7 +282,7 @@ func collectCorpus(t *testing.T, fs *vfs.MemFS, name string) { opts := &pebble.Options{ Comparer: testkeys.Comparer, DisableAutomaticCompactions: true, - FormatMajorVersion: pebble.FormatRangeKeys, + FormatMajorVersion: pebble.FormatMinSupported, FS: fs, MaxManifestFileSize: 96, } diff --git a/scan_internal_test.go b/scan_internal_test.go index 2c3acbc07e6..47964e930dd 100644 --- a/scan_internal_test.go +++ b/scan_internal_test.go @@ -46,7 +46,7 @@ func TestScanStatistics(t *testing.T) { FS: vfs.NewMem(), Logger: testLogger{t: t}, Comparer: testkeys.Comparer, - FormatMajorVersion: FormatRangeKeys, + FormatMajorVersion: FormatMinSupported, BlockPropertyCollectors: []func() BlockPropertyCollector{ sstable.NewTestKeysBlockPropertyCollector, }, diff --git a/snapshot_test.go b/snapshot_test.go index f0514c8d0ba..b9d8f14fcb9 100644 --- a/snapshot_test.go +++ b/snapshot_test.go @@ -62,9 +62,7 @@ func testSnapshotImpl(t *testing.T, newSnapshot func(d *DB) Reader) { defer close() randVersion := func() FormatMajorVersion { - minVersion := formatUnusedPrePebblev1MarkedCompacted - return FormatMajorVersion(int(minVersion) + rand.Intn( - int(internalFormatNewest)-int(minVersion)+1)) + return FormatMinSupported + FormatMajorVersion(rand.Intn(int(internalFormatNewest-FormatMinSupported)+1)) } datadriven.RunTest(t, "testdata/snapshot", func(t *testing.T, td *datadriven.TestData) string { switch td.Cmd { diff --git a/sstable/data_test.go b/sstable/data_test.go index 2b1926a355d..43aa629afd0 100644 --- a/sstable/data_test.go +++ b/sstable/data_test.go @@ -27,11 +27,6 @@ import ( func optsFromArgs(td *datadriven.TestData, writerOpts *WriterOptions) error { for _, arg := range td.CmdArgs { switch arg.Key { - case "leveldb": - if len(arg.Vals) != 0 { - return errors.Errorf("%s: arg %s expects 0 values", td.Cmd, arg.Key) - } - writerOpts.TableFormat = TableFormatLevelDB case "block-size": if len(arg.Vals) != 1 { return errors.Errorf("%s: arg %s expects 1 value", td.Cmd, arg.Key) diff --git a/sstable/format.go b/sstable/format.go index 82310a55bac..0cf74609010 100644 --- a/sstable/format.go +++ b/sstable/format.go @@ -19,15 +19,16 @@ type TableFormat uint32 // Pebble (i.e. the history is linear). const ( TableFormatUnspecified TableFormat = iota - TableFormatLevelDB - TableFormatRocksDBv2 - TableFormatPebblev1 // Block properties. - TableFormatPebblev2 // Range keys. - TableFormatPebblev3 // Value blocks. - TableFormatPebblev4 // DELSIZED tombstones. + TableFormatLevelDB // Deprecated. + TableFormatRocksDBv2 // Deprecated. + TableFormatPebblev1 // Block properties. + TableFormatPebblev2 // Range keys. + TableFormatPebblev3 // Value blocks. + TableFormatPebblev4 // DELSIZED tombstones. NumTableFormats - TableFormatMax = NumTableFormats - 1 + TableFormatMax = NumTableFormats - 1 + TableFormatMinSupported = TableFormatPebblev1 ) // TableFormatPebblev4, in addition to DELSIZED, introduces the use of @@ -185,15 +186,6 @@ const ( // corresponding internal TableFormat. func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) { switch string(magic) { - case levelDBMagic: - return TableFormatLevelDB, nil - case rocksDBMagic: - if version != rocksDBFormatVersion2 { - return TableFormatUnspecified, base.CorruptionErrorf( - "pebble/table: unsupported rocksdb format version %d", errors.Safe(version), - ) - } - return TableFormatRocksDBv2, nil case pebbleDBMagic: switch version { case 1: @@ -219,10 +211,6 @@ func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) { // AsTuple returns the TableFormat's (Magic String, Version) tuple. func (f TableFormat) AsTuple() (string, uint32) { switch f { - case TableFormatLevelDB: - return levelDBMagic, 0 - case TableFormatRocksDBv2: - return rocksDBMagic, 2 case TableFormatPebblev1: return pebbleDBMagic, 1 case TableFormatPebblev2: @@ -239,10 +227,6 @@ func (f TableFormat) AsTuple() (string, uint32) { // String returns the TableFormat (Magic String,Version) tuple. func (f TableFormat) String() string { switch f { - case TableFormatLevelDB: - return "(LevelDB)" - case TableFormatRocksDBv2: - return "(RocksDB,v2)" case TableFormatPebblev1: return "(Pebble,v1)" case TableFormatPebblev2: diff --git a/sstable/format_test.go b/sstable/format_test.go index f5589c12391..be1e911355d 100644 --- a/sstable/format_test.go +++ b/sstable/format_test.go @@ -19,18 +19,6 @@ func TestTableFormat_RoundTrip(t *testing.T) { wantErr string }{ // Valid cases. - { - name: "LevelDB", - magic: levelDBMagic, - version: 0, - want: TableFormatLevelDB, - }, - { - name: "RocksDBv2", - magic: rocksDBMagic, - version: 2, - want: TableFormatRocksDBv2, - }, { name: "PebbleDBv1", magic: pebbleDBMagic, @@ -73,6 +61,16 @@ func TestTableFormat_RoundTrip(t *testing.T) { magic: "foo", wantErr: "pebble/table: invalid table (bad magic number: 0x666f6f)", }, + { + name: "LevelDB", + magic: levelDBMagic, + wantErr: "pebble/table: invalid table (bad magic number: TODO)", + }, + { + name: "RocksDBv2", + magic: rocksDBMagic, + wantErr: "pebble/table: invalid table (bad magic number: TODO)", + }, } for _, tc := range tcs { diff --git a/sstable/options.go b/sstable/options.go index c5e1f7935f2..7bf3d799073 100644 --- a/sstable/options.go +++ b/sstable/options.go @@ -228,9 +228,7 @@ type WriterOptions struct { MergerName string // TableFormat specifies the format version for writing sstables. The default - // is TableFormatRocksDBv2 which creates RocksDB compatible sstables. Use - // TableFormatLevelDB to create LevelDB compatible sstable which can be used - // by a wider range of tools and libraries. + // is TableFormatMinSupported. TableFormat TableFormat // IsStrictObsolete is only relevant for >= TableFormatPebblev4. See comment @@ -299,7 +297,7 @@ func (o WriterOptions) ensureDefaults() WriterOptions { // By default, if the table format is not specified, fall back to using the // most compatible format. if o.TableFormat == TableFormatUnspecified { - o.TableFormat = TableFormatRocksDBv2 + o.TableFormat = TableFormatMinSupported } return o } diff --git a/sstable/reader_test.go b/sstable/reader_test.go index fa5237d4959..8b99d9caf96 100644 --- a/sstable/reader_test.go +++ b/sstable/reader_test.go @@ -584,8 +584,6 @@ func TestReaderStats(t *testing.T) { forEveryTableFormat[string](t, [NumTableFormats]string{ TableFormatUnspecified: "", - TableFormatLevelDB: "testdata/readerstats_LevelDB", - TableFormatRocksDBv2: "testdata/readerstats_LevelDB", TableFormatPebblev1: "testdata/readerstats_LevelDB", TableFormatPebblev2: "testdata/readerstats_LevelDB", TableFormatPebblev3: "testdata/readerstats_Pebblev3", @@ -616,8 +614,6 @@ func TestReaderWithBlockPropertyFilter(t *testing.T) { forEveryTableFormat[string](t, [NumTableFormats]string{ TableFormatUnspecified: "", // Block properties unsupported - TableFormatLevelDB: "", // Block properties unsupported - TableFormatRocksDBv2: "", // Block properties unsupported TableFormatPebblev1: "", // Block properties unsupported TableFormatPebblev2: "testdata/reader_bpf/Pebblev2", TableFormatPebblev3: "testdata/reader_bpf/Pebblev3", @@ -1412,7 +1408,7 @@ func TestReader_TableFormat(t *testing.T) { require.Equal(t, want, got) } - for tf := TableFormatLevelDB; tf <= TableFormatMax; tf++ { + for tf := TableFormatMinSupported; tf <= TableFormatMax; tf++ { t.Run(tf.String(), func(t *testing.T) { test(t, tf) }) diff --git a/sstable/table.go b/sstable/table.go index 2f36f3076e5..8f3c139dadc 100644 --- a/sstable/table.go +++ b/sstable/table.go @@ -206,9 +206,6 @@ const ( minFooterLen = levelDBFooterLen maxFooterLen = rocksDBFooterLen - levelDBFormatVersion = 0 - rocksDBFormatVersion2 = 2 - metaRangeKeyName = "pebble.range_key" metaValueIndexName = "pebble.value_index" metaPropertiesName = "rocksdb.properties" @@ -342,18 +339,7 @@ func readFooter(f objstorage.Readable) (footer, error) { } switch magic := buf[len(buf)-len(rocksDBMagic):]; string(magic) { - case levelDBMagic: - if len(buf) < levelDBFooterLen { - return footer, base.CorruptionErrorf( - "pebble/table: invalid table (footer too short): %d", errors.Safe(len(buf))) - } - footer.footerBH.Offset = uint64(off+int64(len(buf))) - levelDBFooterLen - buf = buf[len(buf)-levelDBFooterLen:] - footer.footerBH.Length = uint64(len(buf)) - footer.format = TableFormatLevelDB - footer.checksum = ChecksumTypeCRC32c - - case rocksDBMagic, pebbleDBMagic: + case pebbleDBMagic: // NOTE: The Pebble magic string implies the same footer format as that used // by the RocksDBv2 table format. if len(buf) < rocksDBFooterLen { @@ -404,14 +390,7 @@ func readFooter(f objstorage.Readable) (footer, error) { func (f footer) encode(buf []byte) []byte { switch magic, version := f.format.AsTuple(); magic { - case levelDBMagic: - buf = buf[:levelDBFooterLen] - clear(buf) - n := encodeBlockHandle(buf[0:], f.metaindexBH) - encodeBlockHandle(buf[n:], f.indexBH) - copy(buf[len(buf)-len(levelDBMagic):], levelDBMagic) - - case rocksDBMagic, pebbleDBMagic: + case pebbleDBMagic: buf = buf[:rocksDBFooterLen] clear(buf) switch f.checksum { @@ -440,12 +419,5 @@ func (f footer) encode(buf []byte) []byte { } func supportsTwoLevelIndex(format TableFormat) bool { - switch format { - case TableFormatLevelDB: - return false - case TableFormatRocksDBv2, TableFormatPebblev1, TableFormatPebblev2, TableFormatPebblev3, TableFormatPebblev4: - return true - default: - panic("sstable: unspecified table format version") - } + return format >= TableFormatMinSupported } diff --git a/sstable/table_test.go b/sstable/table_test.go index 19fffefcce8..d0d3ecd57b2 100644 --- a/sstable/table_test.go +++ b/sstable/table_test.go @@ -716,10 +716,10 @@ func TestMetaIndexEntriesSorted(t *testing.T) { func TestFooterRoundTrip(t *testing.T) { buf := make([]byte, 100+maxFooterLen) - for format := TableFormatLevelDB; format < TableFormatMax; format++ { + for format := TableFormatMinSupported; format < TableFormatMax; format++ { t.Run(fmt.Sprintf("format=%s", format), func(t *testing.T) { checksums := []ChecksumType{ChecksumTypeCRC32c} - if format != TableFormatLevelDB { + if format != TableFormatMinSupported { checksums = []ChecksumType{ChecksumTypeCRC32c, ChecksumTypeXXHash64} } for _, checksum := range checksums { @@ -770,14 +770,6 @@ func TestFooterRoundTrip(t *testing.T) { } func TestReadFooter(t *testing.T) { - encode := func(format TableFormat, checksum ChecksumType) string { - f := footer{ - format: format, - checksum: checksum, - } - return string(f.encode(make([]byte, maxFooterLen))) - } - testCases := []struct { encoded string expected string @@ -785,10 +777,6 @@ func TestReadFooter(t *testing.T) { {strings.Repeat("a", minFooterLen-1), "file size is too small"}, {strings.Repeat("a", levelDBFooterLen), "bad magic number"}, {strings.Repeat("a", rocksDBFooterLen), "bad magic number"}, - {encode(TableFormatLevelDB, 0)[1:], "file size is too small"}, - {encode(TableFormatRocksDBv2, 0)[1:], "footer too short"}, - {encode(TableFormatRocksDBv2, ChecksumTypeNone), "unsupported checksum type"}, - {encode(TableFormatRocksDBv2, ChecksumTypeXXHash), "unsupported checksum type"}, } for _, c := range testCases { t.Run("", func(t *testing.T) { diff --git a/sstable/writer_test.go b/sstable/writer_test.go index 20f9e90eecc..5922badde27 100644 --- a/sstable/writer_test.go +++ b/sstable/writer_test.go @@ -822,7 +822,7 @@ func TestWriter_TableFormatCompatibility(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - for tf := TableFormatLevelDB; tf <= TableFormatMax; tf++ { + for tf := TableFormatMinSupported; tf <= TableFormatMax; tf++ { t.Run(tf.String(), func(t *testing.T) { fs := vfs.NewMem() f, err := fs.Create("sst") diff --git a/table_cache_test.go b/table_cache_test.go index cdf8ed67258..e5d65c06b9b 100644 --- a/table_cache_test.go +++ b/table_cache_test.go @@ -1204,7 +1204,7 @@ func TestTableCacheNoSuchFileError(t *testing.T) { _, _, _ = d.Get([]byte("a")) require.NotZero(t, len(logger.fatalMsgs), "no fatal message emitted") require.Equal(t, 1, len(logger.fatalMsgs), "expected one fatal message; got: %v", logger.fatalMsgs) - require.Contains(t, logger.fatalMsgs[0], "directory contains 6 files, 0 unknown, 0 tables, 2 logs, 1 manifests") + require.Contains(t, logger.fatalMsgs[0], "directory contains 7 files, 2 unknown, 0 tables, 2 logs, 1 manifests") } type catchFatalLogger struct { diff --git a/table_stats_test.go b/table_stats_test.go index 3abece9ebe4..1f33a39ec3f 100644 --- a/table_stats_test.go +++ b/table_stats_test.go @@ -34,7 +34,7 @@ func TestTableStats(t *testing.T) { } opts.DisableAutomaticCompactions = true opts.Comparer = testkeys.Comparer - opts.FormatMajorVersion = FormatRangeKeys + opts.FormatMajorVersion = FormatMinSupported d, err := Open("", opts) require.NoError(t, err) diff --git a/testdata/checkpoint b/testdata/checkpoint index 6ace513eb75..ccae4aa9fd4 100644 --- a/testdata/checkpoint +++ b/testdata/checkpoint @@ -5,85 +5,25 @@ open-dir: db lock: db/LOCK open-dir: db open-dir: db -open: db/CURRENT create: db/MANIFEST-000001 sync: db/MANIFEST-000001 -remove: db/temporary.000001.dbtmp -create: db/temporary.000001.dbtmp -sync: db/temporary.000001.dbtmp -close: db/temporary.000001.dbtmp -rename: db/temporary.000001.dbtmp -> db/CURRENT +create: db/marker.manifest.000001.MANIFEST-000001 +close: db/marker.manifest.000001.MANIFEST-000001 sync: db open-dir: db sync: db/MANIFEST-000001 create: db/000002.log sync: db -create: db/marker.manifest.000001.MANIFEST-000001 -close: db/marker.manifest.000001.MANIFEST-000001 -sync: db -create: db/marker.format-version.000001.002 -close: db/marker.format-version.000001.002 -sync: db -remove: db/temporary.000000.dbtmp -create: db/temporary.000000.dbtmp -sync: db/temporary.000000.dbtmp -close: db/temporary.000000.dbtmp -rename: db/temporary.000000.dbtmp -> db/CURRENT -create: db/marker.format-version.000002.003 -close: db/marker.format-version.000002.003 -remove: db/marker.format-version.000001.002 -sync: db -create: db/marker.format-version.000003.004 -close: db/marker.format-version.000003.004 -remove: db/marker.format-version.000002.003 -sync: db -create: db/marker.format-version.000004.005 -close: db/marker.format-version.000004.005 -remove: db/marker.format-version.000003.004 -sync: db -create: db/marker.format-version.000005.006 -close: db/marker.format-version.000005.006 -remove: db/marker.format-version.000004.005 -sync: db -create: db/marker.format-version.000006.007 -close: db/marker.format-version.000006.007 -remove: db/marker.format-version.000005.006 -sync: db -create: db/marker.format-version.000007.008 -close: db/marker.format-version.000007.008 -remove: db/marker.format-version.000006.007 -sync: db -create: db/marker.format-version.000008.009 -close: db/marker.format-version.000008.009 -remove: db/marker.format-version.000007.008 -sync: db -create: db/marker.format-version.000009.010 -close: db/marker.format-version.000009.010 -remove: db/marker.format-version.000008.009 -sync: db -create: db/marker.format-version.000010.011 -close: db/marker.format-version.000010.011 -remove: db/marker.format-version.000009.010 -sync: db -create: db/marker.format-version.000011.012 -close: db/marker.format-version.000011.012 -remove: db/marker.format-version.000010.011 -sync: db -create: db/marker.format-version.000012.013 -close: db/marker.format-version.000012.013 -remove: db/marker.format-version.000011.012 -sync: db -create: db/marker.format-version.000013.014 -close: db/marker.format-version.000013.014 -remove: db/marker.format-version.000012.013 +create: db/marker.format-version.000001.014 +close: db/marker.format-version.000001.014 sync: db -create: db/marker.format-version.000014.015 -close: db/marker.format-version.000014.015 -remove: db/marker.format-version.000013.014 +create: db/marker.format-version.000002.015 +close: db/marker.format-version.000002.015 +remove: db/marker.format-version.000001.014 sync: db -create: db/marker.format-version.000015.016 -close: db/marker.format-version.000015.016 -remove: db/marker.format-version.000014.015 +create: db/marker.format-version.000003.016 +close: db/marker.format-version.000003.016 +remove: db/marker.format-version.000002.015 sync: db create: db/temporary.000003.dbtmp sync: db/temporary.000003.dbtmp @@ -311,11 +251,10 @@ list db 000006.log 000008.log 000010.sst -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000001.MANIFEST-000001 list checkpoints/checkpoint1 @@ -627,11 +566,10 @@ list db 000010.sst 000011.sst 000014.sst -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000001.MANIFEST-000001 diff --git a/testdata/cleaner b/testdata/cleaner index cd96e7d2fb6..8b5f1ca0945 100644 --- a/testdata/cleaner +++ b/testdata/cleaner @@ -8,19 +8,18 @@ open-dir: db_wal lock: db/LOCK open-dir: db open-dir: db -open: db/CURRENT create: db/MANIFEST-000001 sync: db/MANIFEST-000001 -remove: db/temporary.000001.dbtmp -create: db/temporary.000001.dbtmp -sync: db/temporary.000001.dbtmp -close: db/temporary.000001.dbtmp -rename: db/temporary.000001.dbtmp -> db/CURRENT +create: db/marker.manifest.000001.MANIFEST-000001 +close: db/marker.manifest.000001.MANIFEST-000001 sync: db open-dir: db sync: db/MANIFEST-000001 create: db_wal/000002.log sync: db_wal +create: db/marker.format-version.000001.013 +close: db/marker.format-version.000001.013 +sync: db create: db/temporary.000003.dbtmp sync: db/temporary.000003.dbtmp close: db/temporary.000003.dbtmp @@ -67,16 +66,16 @@ sync: db/MANIFEST-000001 mkdir-all: db_wal/archive 0755 rename: db_wal/000004.log -> db_wal/archive/000004.log open: db/000005.sst -read-at(744, 53): db/000005.sst -read-at(707, 37): db/000005.sst -read-at(79, 628): db/000005.sst +read-at(607, 53): db/000005.sst +read-at(570, 37): db/000005.sst +read-at(79, 491): db/000005.sst read-at(52, 27): db/000005.sst open: db/000005.sst close: db/000005.sst open: db/000007.sst -read-at(718, 53): db/000007.sst -read-at(681, 37): db/000007.sst -read-at(53, 628): db/000007.sst +read-at(581, 53): db/000007.sst +read-at(544, 37): db/000007.sst +read-at(53, 491): db/000007.sst read-at(26, 27): db/000007.sst open: db/000007.sst close: db/000007.sst @@ -101,11 +100,12 @@ rename: db/000007.sst -> db/archive/000007.sst list db ---- 000008.sst -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 archive +marker.format-version.000001.013 +marker.manifest.000001.MANIFEST-000001 list db_wal ---- @@ -132,19 +132,18 @@ open-dir: db1_wal lock: db1/LOCK open-dir: db1 open-dir: db1 -open: db1/CURRENT create: db1/MANIFEST-000001 sync: db1/MANIFEST-000001 -remove: db1/temporary.000001.dbtmp -create: db1/temporary.000001.dbtmp -sync: db1/temporary.000001.dbtmp -close: db1/temporary.000001.dbtmp -rename: db1/temporary.000001.dbtmp -> db1/CURRENT +create: db1/marker.manifest.000001.MANIFEST-000001 +close: db1/marker.manifest.000001.MANIFEST-000001 sync: db1 open-dir: db1 sync: db1/MANIFEST-000001 create: db1_wal/000002.log sync: db1_wal +create: db1/marker.format-version.000001.013 +close: db1/marker.format-version.000001.013 +sync: db1 create: db1/temporary.000003.dbtmp sync: db1/temporary.000003.dbtmp close: db1/temporary.000003.dbtmp @@ -202,9 +201,6 @@ open-dir: db1_wal lock: db1/LOCK open-dir: db1 open-dir: db1 -open: db1/CURRENT -read-at(0, 16): db1/CURRENT -close: db1/CURRENT open: db1/MANIFEST-000001 close: db1/MANIFEST-000001 open-dir: db1 @@ -214,11 +210,9 @@ open: db1_wal/000004.log close: db1_wal/000004.log create: db1/MANIFEST-000458 sync: db1/MANIFEST-000458 -remove: db1/temporary.000458.dbtmp -create: db1/temporary.000458.dbtmp -sync: db1/temporary.000458.dbtmp -close: db1/temporary.000458.dbtmp -rename: db1/temporary.000458.dbtmp -> db1/CURRENT +create: db1/marker.manifest.000002.MANIFEST-000458 +close: db1/marker.manifest.000002.MANIFEST-000458 +remove: db1/marker.manifest.000001.MANIFEST-000001 sync: db1 create: db1_wal/000457.log sync: db1_wal @@ -236,8 +230,9 @@ remove: db1/OPTIONS-000003 list db1 ---- 000005.sst -CURRENT LOCK MANIFEST-000001 MANIFEST-000458 OPTIONS-000459 +marker.format-version.000001.013 +marker.manifest.000002.MANIFEST-000458 diff --git a/testdata/compaction_check_ordering b/testdata/compaction_check_ordering index 07e3197deae..cb786169f56 100644 --- a/testdata/compaction_check_ordering +++ b/testdata/compaction_check_ordering @@ -78,7 +78,7 @@ L1 a.SET.1-b.SET.2 b.SET.1-d.SET.4 ---- -OK +L1 files 000001 and 000002 have overlapping ranges: [a#1,SET-b#2,SET] vs [b#1,SET-d#4,SET] check-ordering L1 diff --git a/testdata/compaction_iter b/testdata/compaction_iter deleted file mode 100644 index 32fe720eba4..00000000000 --- a/testdata/compaction_iter +++ /dev/null @@ -1,1218 +0,0 @@ -define -a.SET.1:b ----- - -iter print-snapshot-pinned -first -next ----- -a#1,1:b (not pinned) -. - -define -a.SET.2:c -a.SET.1:b ----- - -iter print-snapshot-pinned -first -next ----- -a#2,1:c (not pinned) -. - -iter print-snapshot-pinned snapshots=0 -first -next ----- -a#2,1:c (not pinned) -. - -iter snapshots=1 -first -next ----- -a#2,1:c -. - -iter print-snapshot-pinned snapshots=2 -first -next -next ----- -a#2,1:c (not pinned) -a#1,1:b (pinned) -. - -define -a.DEL.2: -a.SET.1:b ----- - -iter -first -next ----- -a#2,0: -. - -iter elide-tombstones=true -first ----- -. - -iter print-snapshot-pinned elide-tombstones=true snapshots=2 -first -next -next ----- -a#2,0: (pinned) -a#1,1:b (pinned) -. - -iter print-snapshot-pinned elide-tombstones=true snapshots=1 -first -next ----- -a#2,0: (pinned) -. - -define -a.DEL.2: -a.SET.1:b -b.SET.3:c ----- - -iter print-snapshot-pinned -first -next -next ----- -a#2,0: (not pinned) -b#3,1:c (not pinned) -. - -iter snapshots=1 -first -next -next ----- -a#2,0: -b#3,1:c -. - -iter snapshots=2 -first -next -next -next ----- -a#2,0: -a#1,1:b -b#3,1:c -. - -define -a.SET.1:a -b.SET.2:b -c.SET.3:c ----- - -iter -first -next -next -next ----- -a#1,1:a -b#2,1:b -c#3,1:c -. - -define -a.MERGE.3:d -a.MERGE.2:c -a.SET.1:b -b.MERGE.2:b -b.MERGE.1:a ----- - -iter -first -next -next ----- -a#3,1:bcd[base] -b#2,2:ab -. - -iter snapshots=3 print-snapshot-pinned -first -next -next -next ----- -a#3,2:d (not pinned) -a#2,1:bc[base] (pinned) -b#2,2:ab (not pinned) -. - -define -a.SET.9:b -a.DEL.8: -a.SET.7:d -a.DEL.6: -a.SET.5:f ----- - -iter -first -next ----- -a#9,1:b -. - -iter snapshots=6 -first -next -next ----- -a#9,1:b -a#5,1:f -. - -iter snapshots=7 -first -next -next ----- -a#9,1:b -a#6,0: -. - -iter snapshots=8 -first -next -next ----- -a#9,1:b -a#7,1:d -. - -iter snapshots=9 -first -next -next ----- -a#9,1:b -a#8,0: -. - -iter snapshots=10 -first -next ----- -a#9,1:b -. - -iter snapshots=(5,6,7,8,9) print-snapshot-pinned -first -next -next -next -next -next ----- -a#9,1:b (not pinned) -a#8,0: (pinned) -a#7,1:d (pinned) -a#6,0: (pinned) -a#5,1:f (pinned) -. - -define -a.INVALID.2:b -a.SET.1:c ----- - -iter -first ----- -err=invalid internal key kind: INVALID - -define -a.SET.2:b -a.INVALID.1:c ----- - -iter -first -next ----- -a#2,1:b -err=invalid internal key kind: INVALID - -define -a.MERGE.2:b -a.INVALID.1:c ----- - -iter -first -next ----- -a#2,2:b -err=invalid internal key kind: INVALID - -define -a.INVALID.2:c -a.RANGEDEL.1:d ----- - -iter -first -tombstones ----- -err=invalid internal key kind: INVALID -. - -define -a.MERGE.2:b -a.MERGE.1:c -a.MERGE.0:d ----- - -iter snapshots=(1,2) print-snapshot-pinned -first -next -next -next ----- -a#2,2:b (not pinned) -a#1,2:c (pinned) -a#0,2:d (pinned) -. - -define -a.SET.2:b -a.RANGEDEL.1:c -b.RANGEDEL.4:d -b.SET.2:e -c.SET.3:f ----- - -# NB: Range deletions are always marked as 'not pinned' currently. Extending -# snapshot-pinning statistics to range deletions and range keys is TODO. - -iter print-snapshot-pinned -first -next -next -next -tombstones ----- -a#2,1:b (not pinned) -a#1,15:c (not pinned) -b#4,15:d (not pinned) -. -a-b#1 -b-c#4 -c-d#4 -. - -iter snapshots=2 print-snapshot-pinned -first -next -next -next -tombstones ----- -a#2,1:b (not pinned) -a#1,15:c (not pinned) -b#4,15:d (not pinned) -. -a-b#1 -b-c#4 -b-c#1 -c-d#4 -. - -iter snapshots=3 print-snapshot-pinned -first -next -next -next -next -tombstones ----- -a#2,1:b (not pinned) -a#1,15:c (not pinned) -b#4,15:d (not pinned) -b#2,1:e (pinned) -. -a-b#1 -b-c#4 -b-c#1 -c-d#4 -. - -iter snapshots=4 print-snapshot-pinned -first -next -next -next -next -next -tombstones ----- -a#2,1:b (not pinned) -a#1,15:c (not pinned) -b#4,15:d (not pinned) -b#2,1:e (pinned) -c#3,1:f (pinned) -. -a-b#1 -b-c#4 -b-c#1 -c-d#4 -. - -define -a.RANGEDEL.3:e -b.SET.4:b -c.SET.3:c -d.SET.2:d -e.SET.1:e ----- - -iter -first -next -next -next -next -tombstones ----- -a#3,15:e -b#4,1:b -c#3,1:c -e#1,1:e -. -a-e#3 -. - -define -a.RANGEDEL.3:e -b.MERGE.4:b -c.MERGE.3:c -d.MERGE.2:d -e.MERGE.1:e ----- - -iter -first -next -next -next -next -tombstones ----- -a#3,15:e -b#4,2:b -c#3,2:c -e#1,2:e -. -a-e#3 -. - -define -a.RANGEDEL.3:c -b.MERGE.5:e -b.MERGE.4:d -b.MERGE.2:c -b.MERGE.1:b -d.MERGE.5:c -d.MERGE.4:b -d.RANGEDEL.3:f -d.MERGE.2:e -d.MERGE.1:d ----- - -iter -first -next -next -next -next -tombstones ----- -a#3,15:c -b#5,1:de[base] -d#5,2:bc -d#3,15:f -. -a-c#3 -d-f#3 -. - -define -a.RANGEDEL.3:d -b.RANGEDEL.2:e -c.RANGEDEL.1:f ----- - -iter -first -next -next -next -tombstones ----- -a#3,15:d -b#2,15:e -c#1,15:f -. -a-b#3 -b-c#3 -c-d#3 -d-e#2 -e-f#1 -. - -iter snapshots=2 -first -next -next -next -tombstones ----- -a#3,15:d -b#2,15:e -c#1,15:f -. -a-b#3 -b-c#3 -c-d#3 -c-d#1 -d-e#2 -d-e#1 -e-f#1 -. - -iter snapshots=3 -first -next -next -next -tombstones ----- -a#3,15:d -b#2,15:e -c#1,15:f -. -a-b#3 -b-c#3 -b-c#2 -c-d#3 -c-d#2 -d-e#2 -e-f#1 -. - -iter snapshots=(2,3) -first -next -next -next -tombstones ----- -a#3,15:d -b#2,15:e -c#1,15:f -. -a-b#3 -b-c#3 -b-c#2 -c-d#3 -c-d#2 -c-d#1 -d-e#2 -d-e#1 -e-f#1 -. - -define -a.RANGEDEL.10:k -f.SET.9:f -f.SET.8:f ----- - -iter snapshots=(9,10) -first -next -tombstones f -next -tombstones ----- -a#10,15:k -f#9,1:f -a-f#10 -. -f#8,1:f -f-k#10 -. - -define -f.RANGEDEL.10:k -f.SET.9:f -f.SET.8:f ----- - -iter snapshots=(9,10) -first -next -tombstones f -next -tombstones ----- -f#10,15:k -f#9,1:f -. -f#8,1:f -f-k#10 -. - -define -a.SET.1:a -b.RANGEDEL.2:d -c.RANGEDEL.3:e -d.SET.4:d ----- - -iter -first -next -next -next -tombstones c -tombstones ----- -a#1,1:a -b#2,15:d -c#3,15:e -d#4,1:d -b-c#2 -. -c-d#3 -d-e#3 -. - -iter snapshots=3 -first -next -next -next -tombstones c -tombstones ----- -a#1,1:a -b#2,15:d -c#3,15:e -d#4,1:d -b-c#2 -. -c-d#3 -c-d#2 -d-e#3 -. - -define -a.SET.1:a -b.RANGEDEL.2:d -c.SET.4:d ----- - -iter -first -next -next -tombstones c -tombstones ----- -a#1,1:a -b#2,15:d -c#4,1:d -b-c#2 -. -c-d#2 -. - -define -a.RANGEDEL.2:d -a.SET.2:a -b.SET.2:b -c.SET.2:c ----- - -iter -first -next -next -next -next ----- -a#2,15:d -a#2,1:a -b#2,1:b -c#2,1:c -. - -define -a.SINGLEDEL.1: ----- - -iter -first -next ----- -a#1,7: -. - -iter elide-tombstones=true -first ----- -. - -define -a.SINGLEDEL.2: -a.SINGLEDEL.1: ----- - -iter -first -next ----- -a#2,7: -. - -define -a.SINGLEDEL.3: -a.SINGLEDEL.2: -a.SET.1:a ----- - -iter -first ----- -. - -define -a.SET.3:a -b.SINGLEDEL.2: -b.DEL.1: ----- - -iter -first -next -next ----- -a#3,1:a -b#2,0: -. - -define -a.SINGLEDEL.2: -a.DEL.1: ----- - -iter -first -next ----- -a#2,0: -. - -iter elide-tombstones=true -first ----- -. - -define -a.SINGLEDEL.2: -a.MERGE.1: ----- - -iter -first -next ----- -a#2,0: -. - -iter elide-tombstones=true -first ----- -. - -define -a.SINGLEDEL.2: -a.SET.1:b ----- - -iter -first ----- -. - -define -a.SET.2:b -a.SINGLEDEL.1: ----- - -iter -first -next ----- -a#2,1:b -. - -define -a.MERGE.6:b -a.SINGLEDEL.5: -a.SET.4:a ----- - -iter -first -next ----- -a#6,1:b[base] -. - -# Non-deterministic use of SINGLEDEL where there are two older SETs that have -# not been deleted or single deleted. It is permitted to shadow both. -define -a.MERGE.6:b -a.SINGLEDEL.5: -a.SET.4:a -a.SET.3:a ----- - -iter -first -next ----- -a#6,1:b[base] -. - -define -a.SINGLEDEL.2: -a.SET.1:b -b.SET.3:c ----- - -iter -first -next ----- -b#3,1:c -. - -define -a.SINGLEDEL.3: -a.SET.2:b -a.SET.1:a ----- - -iter -first -next ----- -a#1,1:a -. - -define -a.SINGLEDEL.3: -a.MERGE.2:b -a.MERGE.1:a ----- - -iter -first -next ----- -a#3,0: -. - -define -a.SINGLEDEL.4: -a.SET.3:val -a.SINGLEDEL.2: -a.SET.1:val ----- - -iter -first ----- -. - -iter snapshots=2 -first -next -next ----- -a#2,7: -a#1,1:val -. - -define -a.SINGLEDEL.4: -a.SET.3:val -a.DEL.2: -a.SET.1:val ----- - -iter -first -next ----- -a#2,0: -. - -iter snapshots=2 -first -next -next ----- -a#2,0: -a#1,1:val -. - -iter snapshots=3 -first -next ----- -a#2,0: -. - -iter snapshots=(2,3) -first -next -next ----- -a#2,0: -a#1,1:val -. - -define -a.SINGLEDEL.4: -a.SET.3:c -a.MERGE.2:b -a.SET.1:a ----- - -iter -first -next ----- -a#2,1:ab[base] -. - -iter snapshots=2 -first -next -next ----- -a#2,2:b -a#1,1:a -. - -iter snapshots=3 -first -next ----- -a#2,1:ab[base] -. - -iter snapshots=(2,3,4) -first -next -next -next -next ----- -a#4,7: -a#3,1:c -a#2,2:b -a#1,1:a -. - -define -a.SINGLEDEL.3: -a.RANGEDEL.2:c -a.SET.1:val ----- - -iter -first -next -next -tombstones ----- -a#3,7: -a#2,15:c -. -a-c#2 -. - -define -a.RANGEDEL.3:d -a.DEL.2: -a.SET.1:a -d.DEL.2: ----- - -iter -first -next -next -tombstones ----- -a#3,15:d -d#2,0: -. -a-d#3 -. - -iter snapshots=3 -first -next -next -next ----- -a#3,15:d -a#2,0: -d#2,0: -. - -iter snapshots=2 -first -next -next -next ----- -a#3,15:d -a#1,1:a -d#2,0: -. - -iter snapshots=1 -first -next -next ----- -a#3,15:d -d#2,0: -. - -define -a.MERGE.2:a -b.RANGEDEL.1:c ----- - -iter -first -tombstones a -next -next -tombstones ----- -a#2,2:a -. -b#1,15:c -. -b-c#1 -. - -define -a.MERGE.2:v2 -a.RANGEDEL.1:b -a.MERGE.1:v1 ----- - -iter allow-zero-seqnum=true -first -next -next -next -tombstones ----- -a#2,2:v2 -a#1,15:b -a#0,2:v1 -. -a-b#1 -. - -# Verify that we transform merge+del -> set. - -define -a.MERGE.5:5 -a.DEL.3: -a.MERGE.1:1 ----- - -iter -first -next ----- -a#5,1:5[base] -. - -iter allow-zero-seqnum=true -first -next ----- -a#0,1:5[base] -. - -iter elide-tombstones=true -first -next ----- -a#5,1:5[base] -. - -iter snapshots=2 -first -next -next ----- -a#5,1:5[base] -a#1,2:1 -. - -iter snapshots=2 elide-tombstones=true -first -next -next ----- -a#5,1:5[base] -a#1,2:1 -. - -# Verify that we transform merge+rangedel -> set. This isn't strictly -# necessary, but provides consistency with the behavior for merge+del. - -define -a.RANGEDEL.3:c -b.MERGE.5:5 -b.SET.2:2 -b.MERGE.1:1 ----- - -iter -first -next -next ----- -a#3,15:c -b#5,1:5[base] -. - -iter allow-zero-seqnum=true -first -next -next ----- -a#3,15:c -b#0,1:5[base] -. - -iter snapshots=2 -first -next -next ----- -a#3,15:c -b#5,1:5[base] -b#1,2:1 - -define -a.RANGEDEL.3:c -b.MERGE.5:5 -b.MERGE.2:2 -b.MERGE.1:1 ----- - -iter -first -next -next ----- -a#3,15:c -b#5,1:5[base] -. - -iter snapshots=2 -first -next -next ----- -a#3,15:c -b#5,1:5[base] -b#1,2:1 - -# NB: Zero values are skipped by deletable merger. -define merger=deletable -a.MERGE.4:-2 -a.MERGE.3:-1 -a.MERGE.2:2 -a.MERGE.1:1 -b.MERGE.4:-3 -b.MERGE.3:3 -b.MERGE.2:2 -b.MERGE.1:-2 ----- - -iter -first -next -next ----- -. -. -. - -# Test that range keys are interleaved, and exposed to the fragmenter. - -define -a.SINGLEDEL.4: -a.SET.3:val -a.DEL.2: -a.SET.1:val -c.SET.3:val ----- - -define-range-keys -a-b:{(#3,RANGEKEYSET,@2,foo)} -d-e:{(#3,RANGEKEYSET,@2,foo)} ----- - -iter -first -next -next -next -next -range-keys ----- -a#72057594037927935,21: -a#2,0: -c#3,1:val -d#72057594037927935,21: -. -a-b:{(#3,RANGEKEYSET,@2,foo)} -d-e:{(#3,RANGEKEYSET,@2,foo)} -. diff --git a/testdata/compaction_read_triggered b/testdata/compaction_read_triggered index 5c3c79691b8..51336f0d9e6 100644 --- a/testdata/compaction_read_triggered +++ b/testdata/compaction_read_triggered @@ -20,7 +20,7 @@ show-read-compactions maybe-compact ---- -[JOB 100] compacted(read) L5 [000004] (784B) Score=0.00 + L6 [000005] (784B) Score=0.00 -> L6 [000006] (778B), in 1.0s (2.0s total), output rate 778B/s +[JOB 100] compacted(read) L5 [000004] (647B) Score=0.00 + L6 [000005] (647B) Score=0.00 -> L6 [000006] (641B), in 1.0s (2.0s total), output rate 641B/s show-read-compactions ---- @@ -81,7 +81,7 @@ show-read-compactions maybe-compact ---- -[JOB 100] compacted(read) L5 [000004] (784B) Score=0.00 + L6 [000005] (784B) Score=0.00 -> L6 [000006] (778B), in 1.0s (2.0s total), output rate 778B/s +[JOB 100] compacted(read) L5 [000004] (647B) Score=0.00 + L6 [000005] (647B) Score=0.00 -> L6 [000006] (641B), in 1.0s (2.0s total), output rate 641B/s show-read-compactions ---- diff --git a/testdata/db-stage-1/000003.log b/testdata/db-stage-1/000003.log deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testdata/db-stage-1/CURRENT b/testdata/db-stage-1/CURRENT deleted file mode 100644 index 7ed683d1711..00000000000 --- a/testdata/db-stage-1/CURRENT +++ /dev/null @@ -1 +0,0 @@ -MANIFEST-000001 diff --git a/testdata/db-stage-1/IDENTITY b/testdata/db-stage-1/IDENTITY deleted file mode 100644 index 5d811ef9b88..00000000000 --- a/testdata/db-stage-1/IDENTITY +++ /dev/null @@ -1 +0,0 @@ -153bdcaeced20168-eed9a14937098f89 \ No newline at end of file diff --git a/testdata/db-stage-1/LOCK b/testdata/db-stage-1/LOCK deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testdata/db-stage-1/LOG b/testdata/db-stage-1/LOG deleted file mode 100644 index 17915b9ca7e..00000000000 --- a/testdata/db-stage-1/LOG +++ /dev/null @@ -1,178 +0,0 @@ -2018/06/26-20:25:41.772917 7fffec34f3c0 RocksDB version: 5.13.4 -2018/06/26-20:25:41.773338 7fffec34f3c0 Git sha rocksdb_build_git_sha: -2018/06/26-20:25:41.773341 7fffec34f3c0 Compile date Jun 19 2018 -2018/06/26-20:25:41.773342 7fffec34f3c0 DB SUMMARY -2018/06/26-20:25:41.773378 7fffec34f3c0 SST files in db-stage-1 dir, Total Num: 0, files: -2018/06/26-20:25:41.773380 7fffec34f3c0 Write Ahead Log file in db-stage-1: -2018/06/26-20:25:41.773382 7fffec34f3c0 Options.error_if_exists: 1 -2018/06/26-20:25:41.773385 7fffec34f3c0 Options.create_if_missing: 1 -2018/06/26-20:25:41.773386 7fffec34f3c0 Options.paranoid_checks: 1 -2018/06/26-20:25:41.773387 7fffec34f3c0 Options.env: 0x10b4c1d28 -2018/06/26-20:25:41.773389 7fffec34f3c0 Options.info_log: 0x7ff1f9700330 -2018/06/26-20:25:41.773390 7fffec34f3c0 Options.max_file_opening_threads: 16 -2018/06/26-20:25:41.773391 7fffec34f3c0 Options.statistics: 0x0 -2018/06/26-20:25:41.773392 7fffec34f3c0 Options.use_fsync: 0 -2018/06/26-20:25:41.773393 7fffec34f3c0 Options.max_log_file_size: 0 -2018/06/26-20:25:41.773394 7fffec34f3c0 Options.max_manifest_file_size: 18446744073709551615 -2018/06/26-20:25:41.773395 7fffec34f3c0 Options.log_file_time_to_roll: 0 -2018/06/26-20:25:41.773396 7fffec34f3c0 Options.keep_log_file_num: 1000 -2018/06/26-20:25:41.773397 7fffec34f3c0 Options.recycle_log_file_num: 0 -2018/06/26-20:25:41.773397 7fffec34f3c0 Options.allow_fallocate: 1 -2018/06/26-20:25:41.773398 7fffec34f3c0 Options.allow_mmap_reads: 0 -2018/06/26-20:25:41.773399 7fffec34f3c0 Options.allow_mmap_writes: 0 -2018/06/26-20:25:41.773400 7fffec34f3c0 Options.use_direct_reads: 0 -2018/06/26-20:25:41.773401 7fffec34f3c0 Options.use_direct_io_for_flush_and_compaction: 0 -2018/06/26-20:25:41.773402 7fffec34f3c0 Options.create_missing_column_families: 0 -2018/06/26-20:25:41.773403 7fffec34f3c0 Options.db_log_dir: -2018/06/26-20:25:41.773404 7fffec34f3c0 Options.wal_dir: db-stage-1 -2018/06/26-20:25:41.773405 7fffec34f3c0 Options.table_cache_numshardbits: 6 -2018/06/26-20:25:41.773406 7fffec34f3c0 Options.max_subcompactions: 1 -2018/06/26-20:25:41.773407 7fffec34f3c0 Options.max_background_flushes: -1 -2018/06/26-20:25:41.773408 7fffec34f3c0 Options.WAL_ttl_seconds: 0 -2018/06/26-20:25:41.773409 7fffec34f3c0 Options.WAL_size_limit_MB: 0 -2018/06/26-20:25:41.773410 7fffec34f3c0 Options.manifest_preallocation_size: 4194304 -2018/06/26-20:25:41.773411 7fffec34f3c0 Options.is_fd_close_on_exec: 1 -2018/06/26-20:25:41.773412 7fffec34f3c0 Options.advise_random_on_open: 1 -2018/06/26-20:25:41.773413 7fffec34f3c0 Options.db_write_buffer_size: 0 -2018/06/26-20:25:41.773414 7fffec34f3c0 Options.write_buffer_manager: 0x7ff1f97003a0 -2018/06/26-20:25:41.773415 7fffec34f3c0 Options.access_hint_on_compaction_start: 1 -2018/06/26-20:25:41.773416 7fffec34f3c0 Options.new_table_reader_for_compaction_inputs: 0 -2018/06/26-20:25:41.773417 7fffec34f3c0 Options.random_access_max_buffer_size: 1048576 -2018/06/26-20:25:41.773418 7fffec34f3c0 Options.use_adaptive_mutex: 0 -2018/06/26-20:25:41.773418 7fffec34f3c0 Options.rate_limiter: 0x0 -2018/06/26-20:25:41.773419 7fffec34f3c0 Options.sst_file_manager.rate_bytes_per_sec: 0 -2018/06/26-20:25:41.773420 7fffec34f3c0 Options.wal_recovery_mode: 2 -2018/06/26-20:25:41.773421 7fffec34f3c0 Options.enable_thread_tracking: 0 -2018/06/26-20:25:41.773422 7fffec34f3c0 Options.enable_pipelined_write: 0 -2018/06/26-20:25:41.773432 7fffec34f3c0 Options.allow_concurrent_memtable_write: 1 -2018/06/26-20:25:41.773433 7fffec34f3c0 Options.enable_write_thread_adaptive_yield: 1 -2018/06/26-20:25:41.773434 7fffec34f3c0 Options.write_thread_max_yield_usec: 100 -2018/06/26-20:25:41.773435 7fffec34f3c0 Options.write_thread_slow_yield_usec: 3 -2018/06/26-20:25:41.773436 7fffec34f3c0 Options.row_cache: None -2018/06/26-20:25:41.773437 7fffec34f3c0 Options.wal_filter: None -2018/06/26-20:25:41.773438 7fffec34f3c0 Options.avoid_flush_during_recovery: 0 -2018/06/26-20:25:41.773439 7fffec34f3c0 Options.allow_ingest_behind: 0 -2018/06/26-20:25:41.773442 7fffec34f3c0 Options.preserve_deletes: 0 -2018/06/26-20:25:41.773443 7fffec34f3c0 Options.two_write_queues: 0 -2018/06/26-20:25:41.773444 7fffec34f3c0 Options.manual_wal_flush: 0 -2018/06/26-20:25:41.773445 7fffec34f3c0 Options.max_background_jobs: 2 -2018/06/26-20:25:41.773446 7fffec34f3c0 Options.max_background_compactions: -1 -2018/06/26-20:25:41.773447 7fffec34f3c0 Options.avoid_flush_during_shutdown: 0 -2018/06/26-20:25:41.773448 7fffec34f3c0 Options.writable_file_max_buffer_size: 1048576 -2018/06/26-20:25:41.773449 7fffec34f3c0 Options.delayed_write_rate : 16777216 -2018/06/26-20:25:41.773450 7fffec34f3c0 Options.max_total_wal_size: 0 -2018/06/26-20:25:41.773451 7fffec34f3c0 Options.delete_obsolete_files_period_micros: 21600000000 -2018/06/26-20:25:41.773452 7fffec34f3c0 Options.stats_dump_period_sec: 600 -2018/06/26-20:25:41.773453 7fffec34f3c0 Options.max_open_files: -1 -2018/06/26-20:25:41.773454 7fffec34f3c0 Options.bytes_per_sync: 0 -2018/06/26-20:25:41.773455 7fffec34f3c0 Options.wal_bytes_per_sync: 0 -2018/06/26-20:25:41.773456 7fffec34f3c0 Options.compaction_readahead_size: 0 -2018/06/26-20:25:41.773457 7fffec34f3c0 Compression algorithms supported: -2018/06/26-20:25:41.773458 7fffec34f3c0 kZSTD supported: 0 -2018/06/26-20:25:41.773459 7fffec34f3c0 kZlibCompression supported: 1 -2018/06/26-20:25:41.773460 7fffec34f3c0 kXpressCompression supported: 0 -2018/06/26-20:25:41.773461 7fffec34f3c0 kSnappyCompression supported: 1 -2018/06/26-20:25:41.773462 7fffec34f3c0 kZSTDNotFinalCompression supported: 0 -2018/06/26-20:25:41.773463 7fffec34f3c0 kLZ4HCCompression supported: 1 -2018/06/26-20:25:41.773464 7fffec34f3c0 kLZ4Compression supported: 1 -2018/06/26-20:25:41.773465 7fffec34f3c0 kBZip2Compression supported: 1 -2018/06/26-20:25:41.773469 7fffec34f3c0 Fast CRC32 supported: Not supported on x86 -2018/06/26-20:25:41.773598 7fffec34f3c0 [db/db_impl_open.cc:227] Creating manifest 1 -2018/06/26-20:25:41.775085 7fffec34f3c0 [db/version_set.cc:3050] Recovering from manifest file: MANIFEST-000001 -2018/06/26-20:25:41.775130 7fffec34f3c0 [db/column_family.cc:433] --------------- Options for column family [default]: -2018/06/26-20:25:41.775133 7fffec34f3c0 Options.comparator: leveldb.BytewiseComparator -2018/06/26-20:25:41.775134 7fffec34f3c0 Options.merge_operator: None -2018/06/26-20:25:41.775135 7fffec34f3c0 Options.compaction_filter: None -2018/06/26-20:25:41.775136 7fffec34f3c0 Options.compaction_filter_factory: None -2018/06/26-20:25:41.775137 7fffec34f3c0 Options.memtable_factory: SkipListFactory -2018/06/26-20:25:41.775138 7fffec34f3c0 Options.table_factory: BlockBasedTable -2018/06/26-20:25:41.775159 7fffec34f3c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7ff1f94efe70) - cache_index_and_filter_blocks: 0 - cache_index_and_filter_blocks_with_high_priority: 0 - pin_l0_filter_and_index_blocks_in_cache: 0 - index_type: 0 - hash_index_allow_collision: 1 - checksum: 1 - no_block_cache: 0 - block_cache: 0x7ff1f94f35e8 - block_cache_name: LRUCache - block_cache_options: - capacity : 8388608 - num_shard_bits : 4 - strict_capacity_limit : 0 - high_pri_pool_ratio: 0.000 - block_cache_compressed: 0x0 - persistent_cache: 0x0 - block_size: 4096 - block_size_deviation: 10 - block_restart_interval: 16 - index_block_restart_interval: 1 - metadata_block_size: 4096 - partition_filters: 0 - use_delta_encoding: 1 - filter_policy: nullptr - whole_key_filtering: 1 - verify_compression: 0 - read_amp_bytes_per_bit: 0 - format_version: 2 - enable_index_compression: 1 -2018/06/26-20:25:41.775174 7fffec34f3c0 Options.write_buffer_size: 67108864 -2018/06/26-20:25:41.775176 7fffec34f3c0 Options.max_write_buffer_number: 2 -2018/06/26-20:25:41.775177 7fffec34f3c0 Options.compression: Snappy -2018/06/26-20:25:41.775178 7fffec34f3c0 Options.bottommost_compression: Disabled -2018/06/26-20:25:41.775179 7fffec34f3c0 Options.prefix_extractor: nullptr -2018/06/26-20:25:41.775180 7fffec34f3c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr -2018/06/26-20:25:41.775181 7fffec34f3c0 Options.num_levels: 7 -2018/06/26-20:25:41.775182 7fffec34f3c0 Options.min_write_buffer_number_to_merge: 1 -2018/06/26-20:25:41.775183 7fffec34f3c0 Options.max_write_buffer_number_to_maintain: 0 -2018/06/26-20:25:41.775184 7fffec34f3c0 Options.compression_opts.window_bits: -14 -2018/06/26-20:25:41.775185 7fffec34f3c0 Options.compression_opts.level: 32767 -2018/06/26-20:25:41.775186 7fffec34f3c0 Options.compression_opts.strategy: 0 -2018/06/26-20:25:41.775187 7fffec34f3c0 Options.compression_opts.max_dict_bytes: 0 -2018/06/26-20:25:41.775188 7fffec34f3c0 Options.level0_file_num_compaction_trigger: 4 -2018/06/26-20:25:41.775189 7fffec34f3c0 Options.level0_slowdown_writes_trigger: 20 -2018/06/26-20:25:41.775190 7fffec34f3c0 Options.level0_stop_writes_trigger: 36 -2018/06/26-20:25:41.775191 7fffec34f3c0 Options.target_file_size_base: 67108864 -2018/06/26-20:25:41.775192 7fffec34f3c0 Options.target_file_size_multiplier: 1 -2018/06/26-20:25:41.775193 7fffec34f3c0 Options.max_bytes_for_level_base: 268435456 -2018/06/26-20:25:41.775194 7fffec34f3c0 Options.level_compaction_dynamic_level_bytes: 0 -2018/06/26-20:25:41.775195 7fffec34f3c0 Options.max_bytes_for_level_multiplier: 10.000000 -2018/06/26-20:25:41.775197 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 -2018/06/26-20:25:41.775198 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 -2018/06/26-20:25:41.775199 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 -2018/06/26-20:25:41.775200 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 -2018/06/26-20:25:41.775201 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 -2018/06/26-20:25:41.775202 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 -2018/06/26-20:25:41.775203 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 -2018/06/26-20:25:41.775203 7fffec34f3c0 Options.max_sequential_skip_in_iterations: 8 -2018/06/26-20:25:41.775204 7fffec34f3c0 Options.max_compaction_bytes: 1677721600 -2018/06/26-20:25:41.775205 7fffec34f3c0 Options.arena_block_size: 8388608 -2018/06/26-20:25:41.775206 7fffec34f3c0 Options.soft_pending_compaction_bytes_limit: 68719476736 -2018/06/26-20:25:41.775207 7fffec34f3c0 Options.hard_pending_compaction_bytes_limit: 274877906944 -2018/06/26-20:25:41.775208 7fffec34f3c0 Options.rate_limit_delay_max_milliseconds: 100 -2018/06/26-20:25:41.775209 7fffec34f3c0 Options.disable_auto_compactions: 0 -2018/06/26-20:25:41.775211 7fffec34f3c0 Options.compaction_style: kCompactionStyleLevel -2018/06/26-20:25:41.775212 7fffec34f3c0 Options.compaction_pri: kByCompensatedSize -2018/06/26-20:25:41.775213 7fffec34f3c0 Options.compaction_options_universal.size_ratio: 1 -2018/06/26-20:25:41.775214 7fffec34f3c0 Options.compaction_options_universal.min_merge_width: 2 -2018/06/26-20:25:41.775215 7fffec34f3c0 Options.compaction_options_universal.max_merge_width: 4294967295 -2018/06/26-20:25:41.775223 7fffec34f3c0 Options.compaction_options_universal.max_size_amplification_percent: 200 -2018/06/26-20:25:41.775225 7fffec34f3c0 Options.compaction_options_universal.compression_size_percent: -1 -2018/06/26-20:25:41.775226 7fffec34f3c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize -2018/06/26-20:25:41.775227 7fffec34f3c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 -2018/06/26-20:25:41.775228 7fffec34f3c0 Options.compaction_options_fifo.allow_compaction: 0 -2018/06/26-20:25:41.775229 7fffec34f3c0 Options.compaction_options_fifo.ttl: 0 -2018/06/26-20:25:41.775230 7fffec34f3c0 Options.table_properties_collectors: -2018/06/26-20:25:41.775231 7fffec34f3c0 Options.inplace_update_support: 0 -2018/06/26-20:25:41.775232 7fffec34f3c0 Options.inplace_update_num_locks: 10000 -2018/06/26-20:25:41.775233 7fffec34f3c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 -2018/06/26-20:25:41.775234 7fffec34f3c0 Options.memtable_huge_page_size: 0 -2018/06/26-20:25:41.775235 7fffec34f3c0 Options.bloom_locality: 0 -2018/06/26-20:25:41.775236 7fffec34f3c0 Options.max_successive_merges: 0 -2018/06/26-20:25:41.775237 7fffec34f3c0 Options.optimize_filters_for_hits: 0 -2018/06/26-20:25:41.775238 7fffec34f3c0 Options.paranoid_file_checks: 0 -2018/06/26-20:25:41.775239 7fffec34f3c0 Options.force_consistency_checks: 0 -2018/06/26-20:25:41.775240 7fffec34f3c0 Options.report_bg_io_stats: 0 -2018/06/26-20:25:41.775874 7fffec34f3c0 [db/version_set.cc:3316] Recovered from manifest file:db-stage-1/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0 -2018/06/26-20:25:41.775878 7fffec34f3c0 [db/version_set.cc:3324] Column family [default] (ID 0), log number is 0 -2018/06/26-20:25:41.778603 7fffec34f3c0 [db/db_impl_open.cc:1188] DB pointer 0x7ff1fa800000 diff --git a/testdata/db-stage-1/MANIFEST-000001 b/testdata/db-stage-1/MANIFEST-000001 deleted file mode 100644 index d89025ae6eb..00000000000 Binary files a/testdata/db-stage-1/MANIFEST-000001 and /dev/null differ diff --git a/testdata/db-stage-1/OPTIONS-000005 b/testdata/db-stage-1/OPTIONS-000005 deleted file mode 100644 index f1212dbf6f2..00000000000 --- a/testdata/db-stage-1/OPTIONS-000005 +++ /dev/null @@ -1,144 +0,0 @@ -# This is a RocksDB option file. -# -# For detailed file format spec, please refer to the example file -# in examples/rocksdb_option_file_example.ini -# - -[Version] - rocksdb_version=5.13.4 - options_file_version=1.1 - -[DBOptions] - allow_mmap_writes=false - base_background_compactions=-1 - new_table_reader_for_compaction_inputs=false - db_log_dir= - wal_recovery_mode=kPointInTimeRecovery - use_direct_reads=false - write_thread_max_yield_usec=100 - max_manifest_file_size=18446744073709551615 - allow_2pc=false - allow_fallocate=true - fail_if_options_file_error=false - allow_ingest_behind=false - allow_mmap_reads=false - skip_log_error_on_recovery=false - recycle_log_file_num=0 - delete_obsolete_files_period_micros=21600000000 - compaction_readahead_size=0 - use_direct_io_for_flush_and_compaction=false - log_file_time_to_roll=0 - create_missing_column_families=false - advise_random_on_open=true - max_log_file_size=0 - stats_dump_period_sec=600 - enable_thread_tracking=false - use_adaptive_mutex=false - create_if_missing=true - is_fd_close_on_exec=true - max_background_flushes=-1 - manifest_preallocation_size=4194304 - error_if_exists=true - skip_stats_update_on_db_open=false - max_open_files=-1 - random_access_max_buffer_size=1048576 - use_fsync=false - max_background_jobs=2 - two_write_queues=false - max_background_compactions=-1 - max_file_opening_threads=16 - table_cache_numshardbits=6 - keep_log_file_num=1000 - avoid_flush_during_shutdown=false - db_write_buffer_size=0 - max_total_wal_size=0 - wal_dir=db-stage-1 - max_subcompactions=1 - WAL_size_limit_MB=0 - paranoid_checks=true - allow_concurrent_memtable_write=true - writable_file_max_buffer_size=1048576 - WAL_ttl_seconds=0 - delayed_write_rate=16777216 - bytes_per_sync=0 - wal_bytes_per_sync=0 - enable_pipelined_write=false - enable_write_thread_adaptive_yield=true - write_thread_slow_yield_usec=3 - access_hint_on_compaction_start=NORMAL - info_log_level=INFO_LEVEL - dump_malloc_stats=false - avoid_flush_during_recovery=false - preserve_deletes=false - manual_wal_flush=false - - -[CFOptions "default"] - report_bg_io_stats=false - inplace_update_support=false - max_compaction_bytes=1677721600 - disable_auto_compactions=false - write_buffer_size=67108864 - bloom_locality=0 - max_bytes_for_level_multiplier=10.000000 - compaction_filter_factory=nullptr - optimize_filters_for_hits=false - target_file_size_base=67108864 - max_write_buffer_number_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - paranoid_file_checks=false - memtable_prefix_bloom_size_ratio=0.000000 - force_consistency_checks=false - max_write_buffer_number=2 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - level0_slowdown_writes_trigger=20 - level_compaction_dynamic_level_bytes=false - compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;} - inplace_update_num_locks=10000 - level0_file_num_compaction_trigger=4 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - num_levels=7 - table_factory=BlockBasedTable - compression_per_level= - target_file_size_multiplier=1 - min_write_buffer_number_to_merge=1 - arena_block_size=8388608 - max_successive_merges=0 - memtable_huge_page_size=0 - compaction_pri=kByCompensatedSize - soft_pending_compaction_bytes_limit=68719476736 - max_bytes_for_level_base=268435456 - comparator=leveldb.BytewiseComparator - max_sequential_skip_in_iterations=8 - bottommost_compression=kDisableCompressionOption - prefix_extractor=nullptr - memtable_insert_with_hint_prefix_extractor=nullptr - memtable_factory=SkipListFactory - compaction_filter=nullptr - compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} - merge_operator=nullptr - compaction_style=kCompactionStyleLevel - -[TableOptions/BlockBasedTable "default"] - format_version=2 - whole_key_filtering=true - verify_compression=false - partition_filters=false - index_block_restart_interval=1 - block_size_deviation=10 - block_size=4096 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - filter_policy=nullptr - metadata_block_size=4096 - no_block_cache=false - checksum=kCRC32c - read_amp_bytes_per_bit=8589934592 - cache_index_and_filter_blocks=false - enable_index_compression=true - index_type=kBinarySearch - hash_index_allow_collision=true - cache_index_and_filter_blocks_with_high_priority=false - flush_block_policy_factory=FlushBlockBySizePolicyFactory - diff --git a/testdata/db-stage-2/000003.log b/testdata/db-stage-2/000003.log deleted file mode 100644 index 5c739c8f048..00000000000 Binary files a/testdata/db-stage-2/000003.log and /dev/null differ diff --git a/testdata/db-stage-2/CURRENT b/testdata/db-stage-2/CURRENT deleted file mode 100644 index 7ed683d1711..00000000000 --- a/testdata/db-stage-2/CURRENT +++ /dev/null @@ -1 +0,0 @@ -MANIFEST-000001 diff --git a/testdata/db-stage-2/IDENTITY b/testdata/db-stage-2/IDENTITY deleted file mode 100644 index 4ac707eaa36..00000000000 --- a/testdata/db-stage-2/IDENTITY +++ /dev/null @@ -1 +0,0 @@ -153bdcaed05b7fc8-eed9a14937098f89 \ No newline at end of file diff --git a/testdata/db-stage-2/LOCK b/testdata/db-stage-2/LOCK deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testdata/db-stage-2/LOG b/testdata/db-stage-2/LOG deleted file mode 100644 index 6a15e602777..00000000000 --- a/testdata/db-stage-2/LOG +++ /dev/null @@ -1,178 +0,0 @@ -2018/06/26-20:25:41.799086 7fffec34f3c0 RocksDB version: 5.13.4 -2018/06/26-20:25:41.799534 7fffec34f3c0 Git sha rocksdb_build_git_sha: -2018/06/26-20:25:41.799538 7fffec34f3c0 Compile date Jun 19 2018 -2018/06/26-20:25:41.799539 7fffec34f3c0 DB SUMMARY -2018/06/26-20:25:41.799573 7fffec34f3c0 SST files in db-stage-2 dir, Total Num: 0, files: -2018/06/26-20:25:41.799576 7fffec34f3c0 Write Ahead Log file in db-stage-2: -2018/06/26-20:25:41.799578 7fffec34f3c0 Options.error_if_exists: 1 -2018/06/26-20:25:41.799581 7fffec34f3c0 Options.create_if_missing: 1 -2018/06/26-20:25:41.799582 7fffec34f3c0 Options.paranoid_checks: 1 -2018/06/26-20:25:41.799583 7fffec34f3c0 Options.env: 0x10df9dd28 -2018/06/26-20:25:41.799584 7fffec34f3c0 Options.info_log: 0x7ff4cd13a240 -2018/06/26-20:25:41.799585 7fffec34f3c0 Options.max_file_opening_threads: 16 -2018/06/26-20:25:41.799586 7fffec34f3c0 Options.statistics: 0x0 -2018/06/26-20:25:41.799587 7fffec34f3c0 Options.use_fsync: 0 -2018/06/26-20:25:41.799588 7fffec34f3c0 Options.max_log_file_size: 0 -2018/06/26-20:25:41.799589 7fffec34f3c0 Options.max_manifest_file_size: 18446744073709551615 -2018/06/26-20:25:41.799590 7fffec34f3c0 Options.log_file_time_to_roll: 0 -2018/06/26-20:25:41.799591 7fffec34f3c0 Options.keep_log_file_num: 1000 -2018/06/26-20:25:41.799592 7fffec34f3c0 Options.recycle_log_file_num: 0 -2018/06/26-20:25:41.799593 7fffec34f3c0 Options.allow_fallocate: 1 -2018/06/26-20:25:41.799594 7fffec34f3c0 Options.allow_mmap_reads: 0 -2018/06/26-20:25:41.799595 7fffec34f3c0 Options.allow_mmap_writes: 0 -2018/06/26-20:25:41.799596 7fffec34f3c0 Options.use_direct_reads: 0 -2018/06/26-20:25:41.799597 7fffec34f3c0 Options.use_direct_io_for_flush_and_compaction: 0 -2018/06/26-20:25:41.799598 7fffec34f3c0 Options.create_missing_column_families: 0 -2018/06/26-20:25:41.799599 7fffec34f3c0 Options.db_log_dir: -2018/06/26-20:25:41.799600 7fffec34f3c0 Options.wal_dir: db-stage-2 -2018/06/26-20:25:41.799601 7fffec34f3c0 Options.table_cache_numshardbits: 6 -2018/06/26-20:25:41.799602 7fffec34f3c0 Options.max_subcompactions: 1 -2018/06/26-20:25:41.799603 7fffec34f3c0 Options.max_background_flushes: -1 -2018/06/26-20:25:41.799604 7fffec34f3c0 Options.WAL_ttl_seconds: 0 -2018/06/26-20:25:41.799605 7fffec34f3c0 Options.WAL_size_limit_MB: 0 -2018/06/26-20:25:41.799606 7fffec34f3c0 Options.manifest_preallocation_size: 4194304 -2018/06/26-20:25:41.799607 7fffec34f3c0 Options.is_fd_close_on_exec: 1 -2018/06/26-20:25:41.799608 7fffec34f3c0 Options.advise_random_on_open: 1 -2018/06/26-20:25:41.799609 7fffec34f3c0 Options.db_write_buffer_size: 0 -2018/06/26-20:25:41.799610 7fffec34f3c0 Options.write_buffer_manager: 0x7ff4cd13a210 -2018/06/26-20:25:41.799611 7fffec34f3c0 Options.access_hint_on_compaction_start: 1 -2018/06/26-20:25:41.799611 7fffec34f3c0 Options.new_table_reader_for_compaction_inputs: 0 -2018/06/26-20:25:41.799612 7fffec34f3c0 Options.random_access_max_buffer_size: 1048576 -2018/06/26-20:25:41.799613 7fffec34f3c0 Options.use_adaptive_mutex: 0 -2018/06/26-20:25:41.799614 7fffec34f3c0 Options.rate_limiter: 0x0 -2018/06/26-20:25:41.799615 7fffec34f3c0 Options.sst_file_manager.rate_bytes_per_sec: 0 -2018/06/26-20:25:41.799616 7fffec34f3c0 Options.wal_recovery_mode: 2 -2018/06/26-20:25:41.799617 7fffec34f3c0 Options.enable_thread_tracking: 0 -2018/06/26-20:25:41.799618 7fffec34f3c0 Options.enable_pipelined_write: 0 -2018/06/26-20:25:41.799628 7fffec34f3c0 Options.allow_concurrent_memtable_write: 1 -2018/06/26-20:25:41.799630 7fffec34f3c0 Options.enable_write_thread_adaptive_yield: 1 -2018/06/26-20:25:41.799631 7fffec34f3c0 Options.write_thread_max_yield_usec: 100 -2018/06/26-20:25:41.799632 7fffec34f3c0 Options.write_thread_slow_yield_usec: 3 -2018/06/26-20:25:41.799633 7fffec34f3c0 Options.row_cache: None -2018/06/26-20:25:41.799634 7fffec34f3c0 Options.wal_filter: None -2018/06/26-20:25:41.799635 7fffec34f3c0 Options.avoid_flush_during_recovery: 0 -2018/06/26-20:25:41.799636 7fffec34f3c0 Options.allow_ingest_behind: 0 -2018/06/26-20:25:41.799639 7fffec34f3c0 Options.preserve_deletes: 0 -2018/06/26-20:25:41.799640 7fffec34f3c0 Options.two_write_queues: 0 -2018/06/26-20:25:41.799641 7fffec34f3c0 Options.manual_wal_flush: 0 -2018/06/26-20:25:41.799642 7fffec34f3c0 Options.max_background_jobs: 2 -2018/06/26-20:25:41.799643 7fffec34f3c0 Options.max_background_compactions: -1 -2018/06/26-20:25:41.799644 7fffec34f3c0 Options.avoid_flush_during_shutdown: 0 -2018/06/26-20:25:41.799645 7fffec34f3c0 Options.writable_file_max_buffer_size: 1048576 -2018/06/26-20:25:41.799646 7fffec34f3c0 Options.delayed_write_rate : 16777216 -2018/06/26-20:25:41.799647 7fffec34f3c0 Options.max_total_wal_size: 0 -2018/06/26-20:25:41.799648 7fffec34f3c0 Options.delete_obsolete_files_period_micros: 21600000000 -2018/06/26-20:25:41.799649 7fffec34f3c0 Options.stats_dump_period_sec: 600 -2018/06/26-20:25:41.799650 7fffec34f3c0 Options.max_open_files: -1 -2018/06/26-20:25:41.799651 7fffec34f3c0 Options.bytes_per_sync: 0 -2018/06/26-20:25:41.799652 7fffec34f3c0 Options.wal_bytes_per_sync: 0 -2018/06/26-20:25:41.799653 7fffec34f3c0 Options.compaction_readahead_size: 0 -2018/06/26-20:25:41.799654 7fffec34f3c0 Compression algorithms supported: -2018/06/26-20:25:41.799655 7fffec34f3c0 kZSTD supported: 0 -2018/06/26-20:25:41.799656 7fffec34f3c0 kZlibCompression supported: 1 -2018/06/26-20:25:41.799657 7fffec34f3c0 kXpressCompression supported: 0 -2018/06/26-20:25:41.799658 7fffec34f3c0 kSnappyCompression supported: 1 -2018/06/26-20:25:41.799659 7fffec34f3c0 kZSTDNotFinalCompression supported: 0 -2018/06/26-20:25:41.799660 7fffec34f3c0 kLZ4HCCompression supported: 1 -2018/06/26-20:25:41.799661 7fffec34f3c0 kLZ4Compression supported: 1 -2018/06/26-20:25:41.799662 7fffec34f3c0 kBZip2Compression supported: 1 -2018/06/26-20:25:41.799666 7fffec34f3c0 Fast CRC32 supported: Not supported on x86 -2018/06/26-20:25:41.799794 7fffec34f3c0 [db/db_impl_open.cc:227] Creating manifest 1 -2018/06/26-20:25:41.800883 7fffec34f3c0 [db/version_set.cc:3050] Recovering from manifest file: MANIFEST-000001 -2018/06/26-20:25:41.800928 7fffec34f3c0 [db/column_family.cc:433] --------------- Options for column family [default]: -2018/06/26-20:25:41.800931 7fffec34f3c0 Options.comparator: leveldb.BytewiseComparator -2018/06/26-20:25:41.800932 7fffec34f3c0 Options.merge_operator: None -2018/06/26-20:25:41.800933 7fffec34f3c0 Options.compaction_filter: None -2018/06/26-20:25:41.800934 7fffec34f3c0 Options.compaction_filter_factory: None -2018/06/26-20:25:41.800935 7fffec34f3c0 Options.memtable_factory: SkipListFactory -2018/06/26-20:25:41.800936 7fffec34f3c0 Options.table_factory: BlockBasedTable -2018/06/26-20:25:41.800955 7fffec34f3c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7ff4cd114640) - cache_index_and_filter_blocks: 0 - cache_index_and_filter_blocks_with_high_priority: 0 - pin_l0_filter_and_index_blocks_in_cache: 0 - index_type: 0 - hash_index_allow_collision: 1 - checksum: 1 - no_block_cache: 0 - block_cache: 0x7ff4cd137428 - block_cache_name: LRUCache - block_cache_options: - capacity : 8388608 - num_shard_bits : 4 - strict_capacity_limit : 0 - high_pri_pool_ratio: 0.000 - block_cache_compressed: 0x0 - persistent_cache: 0x0 - block_size: 4096 - block_size_deviation: 10 - block_restart_interval: 16 - index_block_restart_interval: 1 - metadata_block_size: 4096 - partition_filters: 0 - use_delta_encoding: 1 - filter_policy: nullptr - whole_key_filtering: 1 - verify_compression: 0 - read_amp_bytes_per_bit: 0 - format_version: 2 - enable_index_compression: 1 -2018/06/26-20:25:41.800983 7fffec34f3c0 Options.write_buffer_size: 67108864 -2018/06/26-20:25:41.800984 7fffec34f3c0 Options.max_write_buffer_number: 2 -2018/06/26-20:25:41.800986 7fffec34f3c0 Options.compression: Snappy -2018/06/26-20:25:41.800987 7fffec34f3c0 Options.bottommost_compression: Disabled -2018/06/26-20:25:41.800988 7fffec34f3c0 Options.prefix_extractor: nullptr -2018/06/26-20:25:41.800989 7fffec34f3c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr -2018/06/26-20:25:41.800990 7fffec34f3c0 Options.num_levels: 7 -2018/06/26-20:25:41.800991 7fffec34f3c0 Options.min_write_buffer_number_to_merge: 1 -2018/06/26-20:25:41.800992 7fffec34f3c0 Options.max_write_buffer_number_to_maintain: 0 -2018/06/26-20:25:41.800994 7fffec34f3c0 Options.compression_opts.window_bits: -14 -2018/06/26-20:25:41.800995 7fffec34f3c0 Options.compression_opts.level: 32767 -2018/06/26-20:25:41.800996 7fffec34f3c0 Options.compression_opts.strategy: 0 -2018/06/26-20:25:41.800997 7fffec34f3c0 Options.compression_opts.max_dict_bytes: 0 -2018/06/26-20:25:41.800998 7fffec34f3c0 Options.level0_file_num_compaction_trigger: 4 -2018/06/26-20:25:41.800999 7fffec34f3c0 Options.level0_slowdown_writes_trigger: 20 -2018/06/26-20:25:41.801000 7fffec34f3c0 Options.level0_stop_writes_trigger: 36 -2018/06/26-20:25:41.801001 7fffec34f3c0 Options.target_file_size_base: 67108864 -2018/06/26-20:25:41.801002 7fffec34f3c0 Options.target_file_size_multiplier: 1 -2018/06/26-20:25:41.801003 7fffec34f3c0 Options.max_bytes_for_level_base: 268435456 -2018/06/26-20:25:41.801004 7fffec34f3c0 Options.level_compaction_dynamic_level_bytes: 0 -2018/06/26-20:25:41.801004 7fffec34f3c0 Options.max_bytes_for_level_multiplier: 10.000000 -2018/06/26-20:25:41.801007 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 -2018/06/26-20:25:41.801008 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 -2018/06/26-20:25:41.801009 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 -2018/06/26-20:25:41.801010 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 -2018/06/26-20:25:41.801011 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 -2018/06/26-20:25:41.801012 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 -2018/06/26-20:25:41.801013 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 -2018/06/26-20:25:41.801013 7fffec34f3c0 Options.max_sequential_skip_in_iterations: 8 -2018/06/26-20:25:41.801014 7fffec34f3c0 Options.max_compaction_bytes: 1677721600 -2018/06/26-20:25:41.801015 7fffec34f3c0 Options.arena_block_size: 8388608 -2018/06/26-20:25:41.801016 7fffec34f3c0 Options.soft_pending_compaction_bytes_limit: 68719476736 -2018/06/26-20:25:41.801018 7fffec34f3c0 Options.hard_pending_compaction_bytes_limit: 274877906944 -2018/06/26-20:25:41.801019 7fffec34f3c0 Options.rate_limit_delay_max_milliseconds: 100 -2018/06/26-20:25:41.801020 7fffec34f3c0 Options.disable_auto_compactions: 0 -2018/06/26-20:25:41.801021 7fffec34f3c0 Options.compaction_style: kCompactionStyleLevel -2018/06/26-20:25:41.801022 7fffec34f3c0 Options.compaction_pri: kByCompensatedSize -2018/06/26-20:25:41.801023 7fffec34f3c0 Options.compaction_options_universal.size_ratio: 1 -2018/06/26-20:25:41.801024 7fffec34f3c0 Options.compaction_options_universal.min_merge_width: 2 -2018/06/26-20:25:41.801025 7fffec34f3c0 Options.compaction_options_universal.max_merge_width: 4294967295 -2018/06/26-20:25:41.801033 7fffec34f3c0 Options.compaction_options_universal.max_size_amplification_percent: 200 -2018/06/26-20:25:41.801034 7fffec34f3c0 Options.compaction_options_universal.compression_size_percent: -1 -2018/06/26-20:25:41.801036 7fffec34f3c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize -2018/06/26-20:25:41.801037 7fffec34f3c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 -2018/06/26-20:25:41.801038 7fffec34f3c0 Options.compaction_options_fifo.allow_compaction: 0 -2018/06/26-20:25:41.801039 7fffec34f3c0 Options.compaction_options_fifo.ttl: 0 -2018/06/26-20:25:41.801040 7fffec34f3c0 Options.table_properties_collectors: -2018/06/26-20:25:41.801041 7fffec34f3c0 Options.inplace_update_support: 0 -2018/06/26-20:25:41.801042 7fffec34f3c0 Options.inplace_update_num_locks: 10000 -2018/06/26-20:25:41.801043 7fffec34f3c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 -2018/06/26-20:25:41.801044 7fffec34f3c0 Options.memtable_huge_page_size: 0 -2018/06/26-20:25:41.801045 7fffec34f3c0 Options.bloom_locality: 0 -2018/06/26-20:25:41.801046 7fffec34f3c0 Options.max_successive_merges: 0 -2018/06/26-20:25:41.801047 7fffec34f3c0 Options.optimize_filters_for_hits: 0 -2018/06/26-20:25:41.801048 7fffec34f3c0 Options.paranoid_file_checks: 0 -2018/06/26-20:25:41.801049 7fffec34f3c0 Options.force_consistency_checks: 0 -2018/06/26-20:25:41.801050 7fffec34f3c0 Options.report_bg_io_stats: 0 -2018/06/26-20:25:41.801517 7fffec34f3c0 [db/version_set.cc:3316] Recovered from manifest file:db-stage-2/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0 -2018/06/26-20:25:41.801522 7fffec34f3c0 [db/version_set.cc:3324] Column family [default] (ID 0), log number is 0 -2018/06/26-20:25:41.804127 7fffec34f3c0 [db/db_impl_open.cc:1188] DB pointer 0x7ff4ce041200 diff --git a/testdata/db-stage-2/MANIFEST-000001 b/testdata/db-stage-2/MANIFEST-000001 deleted file mode 100644 index d89025ae6eb..00000000000 Binary files a/testdata/db-stage-2/MANIFEST-000001 and /dev/null differ diff --git a/testdata/db-stage-2/OPTIONS-000005 b/testdata/db-stage-2/OPTIONS-000005 deleted file mode 100644 index a5db5ad5f05..00000000000 --- a/testdata/db-stage-2/OPTIONS-000005 +++ /dev/null @@ -1,144 +0,0 @@ -# This is a RocksDB option file. -# -# For detailed file format spec, please refer to the example file -# in examples/rocksdb_option_file_example.ini -# - -[Version] - rocksdb_version=5.13.4 - options_file_version=1.1 - -[DBOptions] - allow_mmap_writes=false - base_background_compactions=-1 - new_table_reader_for_compaction_inputs=false - db_log_dir= - wal_recovery_mode=kPointInTimeRecovery - use_direct_reads=false - write_thread_max_yield_usec=100 - max_manifest_file_size=18446744073709551615 - allow_2pc=false - allow_fallocate=true - fail_if_options_file_error=false - allow_ingest_behind=false - allow_mmap_reads=false - skip_log_error_on_recovery=false - recycle_log_file_num=0 - delete_obsolete_files_period_micros=21600000000 - compaction_readahead_size=0 - use_direct_io_for_flush_and_compaction=false - log_file_time_to_roll=0 - create_missing_column_families=false - advise_random_on_open=true - max_log_file_size=0 - stats_dump_period_sec=600 - enable_thread_tracking=false - use_adaptive_mutex=false - create_if_missing=true - is_fd_close_on_exec=true - max_background_flushes=-1 - manifest_preallocation_size=4194304 - error_if_exists=true - skip_stats_update_on_db_open=false - max_open_files=-1 - random_access_max_buffer_size=1048576 - use_fsync=false - max_background_jobs=2 - two_write_queues=false - max_background_compactions=-1 - max_file_opening_threads=16 - table_cache_numshardbits=6 - keep_log_file_num=1000 - avoid_flush_during_shutdown=false - db_write_buffer_size=0 - max_total_wal_size=0 - wal_dir=db-stage-2 - max_subcompactions=1 - WAL_size_limit_MB=0 - paranoid_checks=true - allow_concurrent_memtable_write=true - writable_file_max_buffer_size=1048576 - WAL_ttl_seconds=0 - delayed_write_rate=16777216 - bytes_per_sync=0 - wal_bytes_per_sync=0 - enable_pipelined_write=false - enable_write_thread_adaptive_yield=true - write_thread_slow_yield_usec=3 - access_hint_on_compaction_start=NORMAL - info_log_level=INFO_LEVEL - dump_malloc_stats=false - avoid_flush_during_recovery=false - preserve_deletes=false - manual_wal_flush=false - - -[CFOptions "default"] - report_bg_io_stats=false - inplace_update_support=false - max_compaction_bytes=1677721600 - disable_auto_compactions=false - write_buffer_size=67108864 - bloom_locality=0 - max_bytes_for_level_multiplier=10.000000 - compaction_filter_factory=nullptr - optimize_filters_for_hits=false - target_file_size_base=67108864 - max_write_buffer_number_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - paranoid_file_checks=false - memtable_prefix_bloom_size_ratio=0.000000 - force_consistency_checks=false - max_write_buffer_number=2 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - level0_slowdown_writes_trigger=20 - level_compaction_dynamic_level_bytes=false - compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;} - inplace_update_num_locks=10000 - level0_file_num_compaction_trigger=4 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - num_levels=7 - table_factory=BlockBasedTable - compression_per_level= - target_file_size_multiplier=1 - min_write_buffer_number_to_merge=1 - arena_block_size=8388608 - max_successive_merges=0 - memtable_huge_page_size=0 - compaction_pri=kByCompensatedSize - soft_pending_compaction_bytes_limit=68719476736 - max_bytes_for_level_base=268435456 - comparator=leveldb.BytewiseComparator - max_sequential_skip_in_iterations=8 - bottommost_compression=kDisableCompressionOption - prefix_extractor=nullptr - memtable_insert_with_hint_prefix_extractor=nullptr - memtable_factory=SkipListFactory - compaction_filter=nullptr - compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} - merge_operator=nullptr - compaction_style=kCompactionStyleLevel - -[TableOptions/BlockBasedTable "default"] - format_version=2 - whole_key_filtering=true - verify_compression=false - partition_filters=false - index_block_restart_interval=1 - block_size_deviation=10 - block_size=4096 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - filter_policy=nullptr - metadata_block_size=4096 - no_block_cache=false - checksum=kCRC32c - read_amp_bytes_per_bit=8589934592 - cache_index_and_filter_blocks=false - enable_index_compression=true - index_type=kBinarySearch - hash_index_allow_collision=true - cache_index_and_filter_blocks_with_high_priority=false - flush_block_policy_factory=FlushBlockBySizePolicyFactory - diff --git a/testdata/db-stage-3/000004.sst b/testdata/db-stage-3/000004.sst deleted file mode 100644 index c6c2645917c..00000000000 Binary files a/testdata/db-stage-3/000004.sst and /dev/null differ diff --git a/testdata/db-stage-3/000006.log b/testdata/db-stage-3/000006.log deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testdata/db-stage-3/CURRENT b/testdata/db-stage-3/CURRENT deleted file mode 100644 index aa5bb8ea509..00000000000 --- a/testdata/db-stage-3/CURRENT +++ /dev/null @@ -1 +0,0 @@ -MANIFEST-000005 diff --git a/testdata/db-stage-3/IDENTITY b/testdata/db-stage-3/IDENTITY deleted file mode 100644 index 502e86cbe5c..00000000000 --- a/testdata/db-stage-3/IDENTITY +++ /dev/null @@ -1 +0,0 @@ -153bdcaed1e8b360-eed9a14937098f89 \ No newline at end of file diff --git a/testdata/db-stage-3/LOCK b/testdata/db-stage-3/LOCK deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testdata/db-stage-3/LOG b/testdata/db-stage-3/LOG deleted file mode 100644 index fb79e2a7fb9..00000000000 --- a/testdata/db-stage-3/LOG +++ /dev/null @@ -1,185 +0,0 @@ -2018/06/26-20:25:41.831647 7fffec34f3c0 RocksDB version: 5.13.4 -2018/06/26-20:25:41.831673 7fffec34f3c0 Git sha rocksdb_build_git_sha: -2018/06/26-20:25:41.831676 7fffec34f3c0 Compile date Jun 19 2018 -2018/06/26-20:25:41.831679 7fffec34f3c0 DB SUMMARY -2018/06/26-20:25:41.831739 7fffec34f3c0 CURRENT file: CURRENT -2018/06/26-20:25:41.831744 7fffec34f3c0 IDENTITY file: IDENTITY -2018/06/26-20:25:41.831758 7fffec34f3c0 MANIFEST file: MANIFEST-000001 size: 13 Bytes -2018/06/26-20:25:41.831762 7fffec34f3c0 SST files in db-stage-3 dir, Total Num: 0, files: -2018/06/26-20:25:41.831763 7fffec34f3c0 Write Ahead Log file in db-stage-3: 000003.log size: 139 ; -2018/06/26-20:25:41.831767 7fffec34f3c0 Options.error_if_exists: 0 -2018/06/26-20:25:41.831768 7fffec34f3c0 Options.create_if_missing: 0 -2018/06/26-20:25:41.831770 7fffec34f3c0 Options.paranoid_checks: 1 -2018/06/26-20:25:41.831771 7fffec34f3c0 Options.env: 0x10212ad28 -2018/06/26-20:25:41.831773 7fffec34f3c0 Options.info_log: 0x7fbba7b002d0 -2018/06/26-20:25:41.831774 7fffec34f3c0 Options.max_file_opening_threads: 16 -2018/06/26-20:25:41.831776 7fffec34f3c0 Options.statistics: 0x0 -2018/06/26-20:25:41.831777 7fffec34f3c0 Options.use_fsync: 0 -2018/06/26-20:25:41.831779 7fffec34f3c0 Options.max_log_file_size: 0 -2018/06/26-20:25:41.831780 7fffec34f3c0 Options.max_manifest_file_size: 18446744073709551615 -2018/06/26-20:25:41.831782 7fffec34f3c0 Options.log_file_time_to_roll: 0 -2018/06/26-20:25:41.831784 7fffec34f3c0 Options.keep_log_file_num: 1000 -2018/06/26-20:25:41.831785 7fffec34f3c0 Options.recycle_log_file_num: 0 -2018/06/26-20:25:41.831787 7fffec34f3c0 Options.allow_fallocate: 1 -2018/06/26-20:25:41.831788 7fffec34f3c0 Options.allow_mmap_reads: 0 -2018/06/26-20:25:41.831789 7fffec34f3c0 Options.allow_mmap_writes: 0 -2018/06/26-20:25:41.831791 7fffec34f3c0 Options.use_direct_reads: 0 -2018/06/26-20:25:41.831793 7fffec34f3c0 Options.use_direct_io_for_flush_and_compaction: 0 -2018/06/26-20:25:41.831794 7fffec34f3c0 Options.create_missing_column_families: 0 -2018/06/26-20:25:41.831795 7fffec34f3c0 Options.db_log_dir: -2018/06/26-20:25:41.831797 7fffec34f3c0 Options.wal_dir: db-stage-3 -2018/06/26-20:25:41.831799 7fffec34f3c0 Options.table_cache_numshardbits: 6 -2018/06/26-20:25:41.831800 7fffec34f3c0 Options.max_subcompactions: 1 -2018/06/26-20:25:41.831802 7fffec34f3c0 Options.max_background_flushes: -1 -2018/06/26-20:25:41.831803 7fffec34f3c0 Options.WAL_ttl_seconds: 0 -2018/06/26-20:25:41.831806 7fffec34f3c0 Options.WAL_size_limit_MB: 0 -2018/06/26-20:25:41.831807 7fffec34f3c0 Options.manifest_preallocation_size: 4194304 -2018/06/26-20:25:41.831809 7fffec34f3c0 Options.is_fd_close_on_exec: 1 -2018/06/26-20:25:41.831810 7fffec34f3c0 Options.advise_random_on_open: 1 -2018/06/26-20:25:41.831812 7fffec34f3c0 Options.db_write_buffer_size: 0 -2018/06/26-20:25:41.831814 7fffec34f3c0 Options.write_buffer_manager: 0x7fbba7b002a0 -2018/06/26-20:25:41.831815 7fffec34f3c0 Options.access_hint_on_compaction_start: 1 -2018/06/26-20:25:41.831817 7fffec34f3c0 Options.new_table_reader_for_compaction_inputs: 0 -2018/06/26-20:25:41.831829 7fffec34f3c0 Options.random_access_max_buffer_size: 1048576 -2018/06/26-20:25:41.831831 7fffec34f3c0 Options.use_adaptive_mutex: 0 -2018/06/26-20:25:41.831832 7fffec34f3c0 Options.rate_limiter: 0x0 -2018/06/26-20:25:41.831833 7fffec34f3c0 Options.sst_file_manager.rate_bytes_per_sec: 0 -2018/06/26-20:25:41.831834 7fffec34f3c0 Options.wal_recovery_mode: 2 -2018/06/26-20:25:41.831849 7fffec34f3c0 Options.enable_thread_tracking: 0 -2018/06/26-20:25:41.831851 7fffec34f3c0 Options.enable_pipelined_write: 0 -2018/06/26-20:25:41.831851 7fffec34f3c0 Options.allow_concurrent_memtable_write: 1 -2018/06/26-20:25:41.831852 7fffec34f3c0 Options.enable_write_thread_adaptive_yield: 1 -2018/06/26-20:25:41.831853 7fffec34f3c0 Options.write_thread_max_yield_usec: 100 -2018/06/26-20:25:41.831854 7fffec34f3c0 Options.write_thread_slow_yield_usec: 3 -2018/06/26-20:25:41.831855 7fffec34f3c0 Options.row_cache: None -2018/06/26-20:25:41.831856 7fffec34f3c0 Options.wal_filter: None -2018/06/26-20:25:41.831858 7fffec34f3c0 Options.avoid_flush_during_recovery: 0 -2018/06/26-20:25:41.831859 7fffec34f3c0 Options.allow_ingest_behind: 0 -2018/06/26-20:25:41.831860 7fffec34f3c0 Options.preserve_deletes: 0 -2018/06/26-20:25:41.831861 7fffec34f3c0 Options.two_write_queues: 0 -2018/06/26-20:25:41.831862 7fffec34f3c0 Options.manual_wal_flush: 0 -2018/06/26-20:25:41.831863 7fffec34f3c0 Options.max_background_jobs: 2 -2018/06/26-20:25:41.831864 7fffec34f3c0 Options.max_background_compactions: -1 -2018/06/26-20:25:41.831865 7fffec34f3c0 Options.avoid_flush_during_shutdown: 0 -2018/06/26-20:25:41.831866 7fffec34f3c0 Options.writable_file_max_buffer_size: 1048576 -2018/06/26-20:25:41.831867 7fffec34f3c0 Options.delayed_write_rate : 16777216 -2018/06/26-20:25:41.831868 7fffec34f3c0 Options.max_total_wal_size: 0 -2018/06/26-20:25:41.831869 7fffec34f3c0 Options.delete_obsolete_files_period_micros: 21600000000 -2018/06/26-20:25:41.831870 7fffec34f3c0 Options.stats_dump_period_sec: 600 -2018/06/26-20:25:41.831871 7fffec34f3c0 Options.max_open_files: -1 -2018/06/26-20:25:41.831882 7fffec34f3c0 Options.bytes_per_sync: 0 -2018/06/26-20:25:41.831884 7fffec34f3c0 Options.wal_bytes_per_sync: 0 -2018/06/26-20:25:41.831886 7fffec34f3c0 Options.compaction_readahead_size: 0 -2018/06/26-20:25:41.831887 7fffec34f3c0 Compression algorithms supported: -2018/06/26-20:25:41.831889 7fffec34f3c0 kZSTD supported: 0 -2018/06/26-20:25:41.831891 7fffec34f3c0 kZlibCompression supported: 1 -2018/06/26-20:25:41.831893 7fffec34f3c0 kXpressCompression supported: 0 -2018/06/26-20:25:41.831894 7fffec34f3c0 kSnappyCompression supported: 1 -2018/06/26-20:25:41.831896 7fffec34f3c0 kZSTDNotFinalCompression supported: 0 -2018/06/26-20:25:41.831897 7fffec34f3c0 kLZ4HCCompression supported: 1 -2018/06/26-20:25:41.831899 7fffec34f3c0 kLZ4Compression supported: 1 -2018/06/26-20:25:41.831901 7fffec34f3c0 kBZip2Compression supported: 1 -2018/06/26-20:25:41.831903 7fffec34f3c0 Fast CRC32 supported: Not supported on x86 -2018/06/26-20:25:41.832034 7fffec34f3c0 [db/version_set.cc:3050] Recovering from manifest file: MANIFEST-000001 -2018/06/26-20:25:41.832071 7fffec34f3c0 [db/column_family.cc:433] --------------- Options for column family [default]: -2018/06/26-20:25:41.832074 7fffec34f3c0 Options.comparator: leveldb.BytewiseComparator -2018/06/26-20:25:41.832075 7fffec34f3c0 Options.merge_operator: None -2018/06/26-20:25:41.832076 7fffec34f3c0 Options.compaction_filter: None -2018/06/26-20:25:41.832077 7fffec34f3c0 Options.compaction_filter_factory: None -2018/06/26-20:25:41.832078 7fffec34f3c0 Options.memtable_factory: SkipListFactory -2018/06/26-20:25:41.832080 7fffec34f3c0 Options.table_factory: BlockBasedTable -2018/06/26-20:25:41.832100 7fffec34f3c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fbba6700140) - cache_index_and_filter_blocks: 0 - cache_index_and_filter_blocks_with_high_priority: 0 - pin_l0_filter_and_index_blocks_in_cache: 0 - index_type: 0 - hash_index_allow_collision: 1 - checksum: 1 - no_block_cache: 0 - block_cache: 0x7fbba67020f8 - block_cache_name: LRUCache - block_cache_options: - capacity : 8388608 - num_shard_bits : 4 - strict_capacity_limit : 0 - high_pri_pool_ratio: 0.000 - block_cache_compressed: 0x0 - persistent_cache: 0x0 - block_size: 4096 - block_size_deviation: 10 - block_restart_interval: 16 - index_block_restart_interval: 1 - metadata_block_size: 4096 - partition_filters: 0 - use_delta_encoding: 1 - filter_policy: nullptr - whole_key_filtering: 1 - verify_compression: 0 - read_amp_bytes_per_bit: 0 - format_version: 2 - enable_index_compression: 1 -2018/06/26-20:25:41.832118 7fffec34f3c0 Options.write_buffer_size: 67108864 -2018/06/26-20:25:41.832120 7fffec34f3c0 Options.max_write_buffer_number: 2 -2018/06/26-20:25:41.832121 7fffec34f3c0 Options.compression: Snappy -2018/06/26-20:25:41.832122 7fffec34f3c0 Options.bottommost_compression: Disabled -2018/06/26-20:25:41.832124 7fffec34f3c0 Options.prefix_extractor: nullptr -2018/06/26-20:25:41.832125 7fffec34f3c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr -2018/06/26-20:25:41.832126 7fffec34f3c0 Options.num_levels: 7 -2018/06/26-20:25:41.832127 7fffec34f3c0 Options.min_write_buffer_number_to_merge: 1 -2018/06/26-20:25:41.832127 7fffec34f3c0 Options.max_write_buffer_number_to_maintain: 0 -2018/06/26-20:25:41.832128 7fffec34f3c0 Options.compression_opts.window_bits: -14 -2018/06/26-20:25:41.832129 7fffec34f3c0 Options.compression_opts.level: 32767 -2018/06/26-20:25:41.832130 7fffec34f3c0 Options.compression_opts.strategy: 0 -2018/06/26-20:25:41.832131 7fffec34f3c0 Options.compression_opts.max_dict_bytes: 0 -2018/06/26-20:25:41.832132 7fffec34f3c0 Options.level0_file_num_compaction_trigger: 4 -2018/06/26-20:25:41.832133 7fffec34f3c0 Options.level0_slowdown_writes_trigger: 20 -2018/06/26-20:25:41.832134 7fffec34f3c0 Options.level0_stop_writes_trigger: 36 -2018/06/26-20:25:41.832135 7fffec34f3c0 Options.target_file_size_base: 67108864 -2018/06/26-20:25:41.832136 7fffec34f3c0 Options.target_file_size_multiplier: 1 -2018/06/26-20:25:41.832137 7fffec34f3c0 Options.max_bytes_for_level_base: 268435456 -2018/06/26-20:25:41.832138 7fffec34f3c0 Options.level_compaction_dynamic_level_bytes: 0 -2018/06/26-20:25:41.832139 7fffec34f3c0 Options.max_bytes_for_level_multiplier: 10.000000 -2018/06/26-20:25:41.832141 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 -2018/06/26-20:25:41.832142 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 -2018/06/26-20:25:41.832143 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 -2018/06/26-20:25:41.832144 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 -2018/06/26-20:25:41.832145 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 -2018/06/26-20:25:41.832146 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 -2018/06/26-20:25:41.832147 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 -2018/06/26-20:25:41.832148 7fffec34f3c0 Options.max_sequential_skip_in_iterations: 8 -2018/06/26-20:25:41.832149 7fffec34f3c0 Options.max_compaction_bytes: 1677721600 -2018/06/26-20:25:41.832150 7fffec34f3c0 Options.arena_block_size: 8388608 -2018/06/26-20:25:41.832151 7fffec34f3c0 Options.soft_pending_compaction_bytes_limit: 68719476736 -2018/06/26-20:25:41.832152 7fffec34f3c0 Options.hard_pending_compaction_bytes_limit: 274877906944 -2018/06/26-20:25:41.832153 7fffec34f3c0 Options.rate_limit_delay_max_milliseconds: 100 -2018/06/26-20:25:41.832154 7fffec34f3c0 Options.disable_auto_compactions: 0 -2018/06/26-20:25:41.832155 7fffec34f3c0 Options.compaction_style: kCompactionStyleLevel -2018/06/26-20:25:41.832157 7fffec34f3c0 Options.compaction_pri: kByCompensatedSize -2018/06/26-20:25:41.832158 7fffec34f3c0 Options.compaction_options_universal.size_ratio: 1 -2018/06/26-20:25:41.832165 7fffec34f3c0 Options.compaction_options_universal.min_merge_width: 2 -2018/06/26-20:25:41.832167 7fffec34f3c0 Options.compaction_options_universal.max_merge_width: 4294967295 -2018/06/26-20:25:41.832168 7fffec34f3c0 Options.compaction_options_universal.max_size_amplification_percent: 200 -2018/06/26-20:25:41.832169 7fffec34f3c0 Options.compaction_options_universal.compression_size_percent: -1 -2018/06/26-20:25:41.832171 7fffec34f3c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize -2018/06/26-20:25:41.832172 7fffec34f3c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 -2018/06/26-20:25:41.832173 7fffec34f3c0 Options.compaction_options_fifo.allow_compaction: 0 -2018/06/26-20:25:41.832173 7fffec34f3c0 Options.compaction_options_fifo.ttl: 0 -2018/06/26-20:25:41.832174 7fffec34f3c0 Options.table_properties_collectors: -2018/06/26-20:25:41.832175 7fffec34f3c0 Options.inplace_update_support: 0 -2018/06/26-20:25:41.832176 7fffec34f3c0 Options.inplace_update_num_locks: 10000 -2018/06/26-20:25:41.832177 7fffec34f3c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 -2018/06/26-20:25:41.832179 7fffec34f3c0 Options.memtable_huge_page_size: 0 -2018/06/26-20:25:41.832180 7fffec34f3c0 Options.bloom_locality: 0 -2018/06/26-20:25:41.832181 7fffec34f3c0 Options.max_successive_merges: 0 -2018/06/26-20:25:41.832181 7fffec34f3c0 Options.optimize_filters_for_hits: 0 -2018/06/26-20:25:41.832182 7fffec34f3c0 Options.paranoid_file_checks: 0 -2018/06/26-20:25:41.832183 7fffec34f3c0 Options.force_consistency_checks: 0 -2018/06/26-20:25:41.832184 7fffec34f3c0 Options.report_bg_io_stats: 0 -2018/06/26-20:25:41.832689 7fffec34f3c0 [db/version_set.cc:3316] Recovered from manifest file:db-stage-3/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0 -2018/06/26-20:25:41.832695 7fffec34f3c0 [db/version_set.cc:3324] Column family [default] (ID 0), log number is 0 -2018/06/26-20:25:41.832775 7fffec34f3c0 EVENT_LOG_v1 {"time_micros": 1530059141832760, "job": 1, "event": "recovery_started", "log_files": [3]} -2018/06/26-20:25:41.832780 7fffec34f3c0 [db/db_impl_open.cc:541] Recovering log #3 mode 2 -2018/06/26-20:25:41.834609 7fffec34f3c0 EVENT_LOG_v1 {"time_micros": 1530059141834592, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 986, "table_properties": {"data_size": 62, "index_size": 27, "filter_size": 0, "raw_key_size": 33, "raw_average_key_size": 11, "raw_value_size": 9, "raw_average_value_size": 3, "num_data_blocks": 1, "num_entries": 3, "filter_policy_name": "", "kDeletedKeys": "1", "kMergeOperands": "0"}} -2018/06/26-20:25:41.834659 7fffec34f3c0 [db/version_set.cc:2825] Creating manifest 5 -2018/06/26-20:25:41.835739 7fffec34f3c0 EVENT_LOG_v1 {"time_micros": 1530059141835732, "job": 1, "event": "recovery_finished"} -2018/06/26-20:25:41.839675 7fffec34f3c0 [db/db_impl_open.cc:1188] DB pointer 0x7fbba9013e00 diff --git a/testdata/db-stage-3/LOG.old.1530059141831082 b/testdata/db-stage-3/LOG.old.1530059141831082 deleted file mode 100644 index 502e94e6248..00000000000 --- a/testdata/db-stage-3/LOG.old.1530059141831082 +++ /dev/null @@ -1,180 +0,0 @@ -2018/06/26-20:25:41.824856 7fffec34f3c0 RocksDB version: 5.13.4 -2018/06/26-20:25:41.825422 7fffec34f3c0 Git sha rocksdb_build_git_sha: -2018/06/26-20:25:41.825425 7fffec34f3c0 Compile date Jun 19 2018 -2018/06/26-20:25:41.825427 7fffec34f3c0 DB SUMMARY -2018/06/26-20:25:41.825463 7fffec34f3c0 SST files in db-stage-3 dir, Total Num: 0, files: -2018/06/26-20:25:41.825466 7fffec34f3c0 Write Ahead Log file in db-stage-3: -2018/06/26-20:25:41.825467 7fffec34f3c0 Options.error_if_exists: 1 -2018/06/26-20:25:41.825470 7fffec34f3c0 Options.create_if_missing: 1 -2018/06/26-20:25:41.825472 7fffec34f3c0 Options.paranoid_checks: 1 -2018/06/26-20:25:41.825473 7fffec34f3c0 Options.env: 0x10212ad28 -2018/06/26-20:25:41.825474 7fffec34f3c0 Options.info_log: 0x7fbba6705230 -2018/06/26-20:25:41.825475 7fffec34f3c0 Options.max_file_opening_threads: 16 -2018/06/26-20:25:41.825476 7fffec34f3c0 Options.statistics: 0x0 -2018/06/26-20:25:41.825477 7fffec34f3c0 Options.use_fsync: 0 -2018/06/26-20:25:41.825478 7fffec34f3c0 Options.max_log_file_size: 0 -2018/06/26-20:25:41.825479 7fffec34f3c0 Options.max_manifest_file_size: 18446744073709551615 -2018/06/26-20:25:41.825480 7fffec34f3c0 Options.log_file_time_to_roll: 0 -2018/06/26-20:25:41.825481 7fffec34f3c0 Options.keep_log_file_num: 1000 -2018/06/26-20:25:41.825482 7fffec34f3c0 Options.recycle_log_file_num: 0 -2018/06/26-20:25:41.825483 7fffec34f3c0 Options.allow_fallocate: 1 -2018/06/26-20:25:41.825484 7fffec34f3c0 Options.allow_mmap_reads: 0 -2018/06/26-20:25:41.825485 7fffec34f3c0 Options.allow_mmap_writes: 0 -2018/06/26-20:25:41.825485 7fffec34f3c0 Options.use_direct_reads: 0 -2018/06/26-20:25:41.825486 7fffec34f3c0 Options.use_direct_io_for_flush_and_compaction: 0 -2018/06/26-20:25:41.825487 7fffec34f3c0 Options.create_missing_column_families: 0 -2018/06/26-20:25:41.825488 7fffec34f3c0 Options.db_log_dir: -2018/06/26-20:25:41.825489 7fffec34f3c0 Options.wal_dir: db-stage-3 -2018/06/26-20:25:41.825490 7fffec34f3c0 Options.table_cache_numshardbits: 6 -2018/06/26-20:25:41.825491 7fffec34f3c0 Options.max_subcompactions: 1 -2018/06/26-20:25:41.825492 7fffec34f3c0 Options.max_background_flushes: -1 -2018/06/26-20:25:41.825493 7fffec34f3c0 Options.WAL_ttl_seconds: 0 -2018/06/26-20:25:41.825494 7fffec34f3c0 Options.WAL_size_limit_MB: 0 -2018/06/26-20:25:41.825495 7fffec34f3c0 Options.manifest_preallocation_size: 4194304 -2018/06/26-20:25:41.825496 7fffec34f3c0 Options.is_fd_close_on_exec: 1 -2018/06/26-20:25:41.825497 7fffec34f3c0 Options.advise_random_on_open: 1 -2018/06/26-20:25:41.825498 7fffec34f3c0 Options.db_write_buffer_size: 0 -2018/06/26-20:25:41.825499 7fffec34f3c0 Options.write_buffer_manager: 0x7fbba6705200 -2018/06/26-20:25:41.825500 7fffec34f3c0 Options.access_hint_on_compaction_start: 1 -2018/06/26-20:25:41.825501 7fffec34f3c0 Options.new_table_reader_for_compaction_inputs: 0 -2018/06/26-20:25:41.825502 7fffec34f3c0 Options.random_access_max_buffer_size: 1048576 -2018/06/26-20:25:41.825503 7fffec34f3c0 Options.use_adaptive_mutex: 0 -2018/06/26-20:25:41.825504 7fffec34f3c0 Options.rate_limiter: 0x0 -2018/06/26-20:25:41.825505 7fffec34f3c0 Options.sst_file_manager.rate_bytes_per_sec: 0 -2018/06/26-20:25:41.825506 7fffec34f3c0 Options.wal_recovery_mode: 2 -2018/06/26-20:25:41.825507 7fffec34f3c0 Options.enable_thread_tracking: 0 -2018/06/26-20:25:41.825507 7fffec34f3c0 Options.enable_pipelined_write: 0 -2018/06/26-20:25:41.825517 7fffec34f3c0 Options.allow_concurrent_memtable_write: 1 -2018/06/26-20:25:41.825519 7fffec34f3c0 Options.enable_write_thread_adaptive_yield: 1 -2018/06/26-20:25:41.825520 7fffec34f3c0 Options.write_thread_max_yield_usec: 100 -2018/06/26-20:25:41.825521 7fffec34f3c0 Options.write_thread_slow_yield_usec: 3 -2018/06/26-20:25:41.825522 7fffec34f3c0 Options.row_cache: None -2018/06/26-20:25:41.825523 7fffec34f3c0 Options.wal_filter: None -2018/06/26-20:25:41.825524 7fffec34f3c0 Options.avoid_flush_during_recovery: 0 -2018/06/26-20:25:41.825525 7fffec34f3c0 Options.allow_ingest_behind: 0 -2018/06/26-20:25:41.825528 7fffec34f3c0 Options.preserve_deletes: 0 -2018/06/26-20:25:41.825529 7fffec34f3c0 Options.two_write_queues: 0 -2018/06/26-20:25:41.825530 7fffec34f3c0 Options.manual_wal_flush: 0 -2018/06/26-20:25:41.825531 7fffec34f3c0 Options.max_background_jobs: 2 -2018/06/26-20:25:41.825532 7fffec34f3c0 Options.max_background_compactions: -1 -2018/06/26-20:25:41.825533 7fffec34f3c0 Options.avoid_flush_during_shutdown: 0 -2018/06/26-20:25:41.825534 7fffec34f3c0 Options.writable_file_max_buffer_size: 1048576 -2018/06/26-20:25:41.825536 7fffec34f3c0 Options.delayed_write_rate : 16777216 -2018/06/26-20:25:41.825538 7fffec34f3c0 Options.max_total_wal_size: 0 -2018/06/26-20:25:41.825539 7fffec34f3c0 Options.delete_obsolete_files_period_micros: 21600000000 -2018/06/26-20:25:41.825541 7fffec34f3c0 Options.stats_dump_period_sec: 600 -2018/06/26-20:25:41.825543 7fffec34f3c0 Options.max_open_files: -1 -2018/06/26-20:25:41.825544 7fffec34f3c0 Options.bytes_per_sync: 0 -2018/06/26-20:25:41.825545 7fffec34f3c0 Options.wal_bytes_per_sync: 0 -2018/06/26-20:25:41.825546 7fffec34f3c0 Options.compaction_readahead_size: 0 -2018/06/26-20:25:41.825547 7fffec34f3c0 Compression algorithms supported: -2018/06/26-20:25:41.825548 7fffec34f3c0 kZSTD supported: 0 -2018/06/26-20:25:41.825549 7fffec34f3c0 kZlibCompression supported: 1 -2018/06/26-20:25:41.825550 7fffec34f3c0 kXpressCompression supported: 0 -2018/06/26-20:25:41.825551 7fffec34f3c0 kSnappyCompression supported: 1 -2018/06/26-20:25:41.825552 7fffec34f3c0 kZSTDNotFinalCompression supported: 0 -2018/06/26-20:25:41.825553 7fffec34f3c0 kLZ4HCCompression supported: 1 -2018/06/26-20:25:41.825554 7fffec34f3c0 kLZ4Compression supported: 1 -2018/06/26-20:25:41.825555 7fffec34f3c0 kBZip2Compression supported: 1 -2018/06/26-20:25:41.825559 7fffec34f3c0 Fast CRC32 supported: Not supported on x86 -2018/06/26-20:25:41.825731 7fffec34f3c0 [db/db_impl_open.cc:227] Creating manifest 1 -2018/06/26-20:25:41.826996 7fffec34f3c0 [db/version_set.cc:3050] Recovering from manifest file: MANIFEST-000001 -2018/06/26-20:25:41.827053 7fffec34f3c0 [db/column_family.cc:433] --------------- Options for column family [default]: -2018/06/26-20:25:41.827055 7fffec34f3c0 Options.comparator: leveldb.BytewiseComparator -2018/06/26-20:25:41.827057 7fffec34f3c0 Options.merge_operator: None -2018/06/26-20:25:41.827058 7fffec34f3c0 Options.compaction_filter: None -2018/06/26-20:25:41.827059 7fffec34f3c0 Options.compaction_filter_factory: None -2018/06/26-20:25:41.827060 7fffec34f3c0 Options.memtable_factory: SkipListFactory -2018/06/26-20:25:41.827061 7fffec34f3c0 Options.table_factory: BlockBasedTable -2018/06/26-20:25:41.827090 7fffec34f3c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7fbba6700140) - cache_index_and_filter_blocks: 0 - cache_index_and_filter_blocks_with_high_priority: 0 - pin_l0_filter_and_index_blocks_in_cache: 0 - index_type: 0 - hash_index_allow_collision: 1 - checksum: 1 - no_block_cache: 0 - block_cache: 0x7fbba67020f8 - block_cache_name: LRUCache - block_cache_options: - capacity : 8388608 - num_shard_bits : 4 - strict_capacity_limit : 0 - high_pri_pool_ratio: 0.000 - block_cache_compressed: 0x0 - persistent_cache: 0x0 - block_size: 4096 - block_size_deviation: 10 - block_restart_interval: 16 - index_block_restart_interval: 1 - metadata_block_size: 4096 - partition_filters: 0 - use_delta_encoding: 1 - filter_policy: nullptr - whole_key_filtering: 1 - verify_compression: 0 - read_amp_bytes_per_bit: 0 - format_version: 2 - enable_index_compression: 1 -2018/06/26-20:25:41.827112 7fffec34f3c0 Options.write_buffer_size: 67108864 -2018/06/26-20:25:41.827114 7fffec34f3c0 Options.max_write_buffer_number: 2 -2018/06/26-20:25:41.827116 7fffec34f3c0 Options.compression: Snappy -2018/06/26-20:25:41.827117 7fffec34f3c0 Options.bottommost_compression: Disabled -2018/06/26-20:25:41.827118 7fffec34f3c0 Options.prefix_extractor: nullptr -2018/06/26-20:25:41.827119 7fffec34f3c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr -2018/06/26-20:25:41.827120 7fffec34f3c0 Options.num_levels: 7 -2018/06/26-20:25:41.827121 7fffec34f3c0 Options.min_write_buffer_number_to_merge: 1 -2018/06/26-20:25:41.827122 7fffec34f3c0 Options.max_write_buffer_number_to_maintain: 0 -2018/06/26-20:25:41.827123 7fffec34f3c0 Options.compression_opts.window_bits: -14 -2018/06/26-20:25:41.827124 7fffec34f3c0 Options.compression_opts.level: 32767 -2018/06/26-20:25:41.827125 7fffec34f3c0 Options.compression_opts.strategy: 0 -2018/06/26-20:25:41.827126 7fffec34f3c0 Options.compression_opts.max_dict_bytes: 0 -2018/06/26-20:25:41.827127 7fffec34f3c0 Options.level0_file_num_compaction_trigger: 4 -2018/06/26-20:25:41.827128 7fffec34f3c0 Options.level0_slowdown_writes_trigger: 20 -2018/06/26-20:25:41.827129 7fffec34f3c0 Options.level0_stop_writes_trigger: 36 -2018/06/26-20:25:41.827130 7fffec34f3c0 Options.target_file_size_base: 67108864 -2018/06/26-20:25:41.827131 7fffec34f3c0 Options.target_file_size_multiplier: 1 -2018/06/26-20:25:41.827132 7fffec34f3c0 Options.max_bytes_for_level_base: 268435456 -2018/06/26-20:25:41.827133 7fffec34f3c0 Options.level_compaction_dynamic_level_bytes: 0 -2018/06/26-20:25:41.827134 7fffec34f3c0 Options.max_bytes_for_level_multiplier: 10.000000 -2018/06/26-20:25:41.827136 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 -2018/06/26-20:25:41.827137 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 -2018/06/26-20:25:41.827138 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 -2018/06/26-20:25:41.827139 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 -2018/06/26-20:25:41.827140 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 -2018/06/26-20:25:41.827141 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 -2018/06/26-20:25:41.827142 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 -2018/06/26-20:25:41.827143 7fffec34f3c0 Options.max_sequential_skip_in_iterations: 8 -2018/06/26-20:25:41.827144 7fffec34f3c0 Options.max_compaction_bytes: 1677721600 -2018/06/26-20:25:41.827145 7fffec34f3c0 Options.arena_block_size: 8388608 -2018/06/26-20:25:41.827146 7fffec34f3c0 Options.soft_pending_compaction_bytes_limit: 68719476736 -2018/06/26-20:25:41.827147 7fffec34f3c0 Options.hard_pending_compaction_bytes_limit: 274877906944 -2018/06/26-20:25:41.827148 7fffec34f3c0 Options.rate_limit_delay_max_milliseconds: 100 -2018/06/26-20:25:41.827149 7fffec34f3c0 Options.disable_auto_compactions: 0 -2018/06/26-20:25:41.827151 7fffec34f3c0 Options.compaction_style: kCompactionStyleLevel -2018/06/26-20:25:41.827152 7fffec34f3c0 Options.compaction_pri: kByCompensatedSize -2018/06/26-20:25:41.827153 7fffec34f3c0 Options.compaction_options_universal.size_ratio: 1 -2018/06/26-20:25:41.827154 7fffec34f3c0 Options.compaction_options_universal.min_merge_width: 2 -2018/06/26-20:25:41.827155 7fffec34f3c0 Options.compaction_options_universal.max_merge_width: 4294967295 -2018/06/26-20:25:41.827164 7fffec34f3c0 Options.compaction_options_universal.max_size_amplification_percent: 200 -2018/06/26-20:25:41.827166 7fffec34f3c0 Options.compaction_options_universal.compression_size_percent: -1 -2018/06/26-20:25:41.827167 7fffec34f3c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize -2018/06/26-20:25:41.827168 7fffec34f3c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 -2018/06/26-20:25:41.827169 7fffec34f3c0 Options.compaction_options_fifo.allow_compaction: 0 -2018/06/26-20:25:41.827170 7fffec34f3c0 Options.compaction_options_fifo.ttl: 0 -2018/06/26-20:25:41.827171 7fffec34f3c0 Options.table_properties_collectors: -2018/06/26-20:25:41.827172 7fffec34f3c0 Options.inplace_update_support: 0 -2018/06/26-20:25:41.827173 7fffec34f3c0 Options.inplace_update_num_locks: 10000 -2018/06/26-20:25:41.827174 7fffec34f3c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 -2018/06/26-20:25:41.827175 7fffec34f3c0 Options.memtable_huge_page_size: 0 -2018/06/26-20:25:41.827176 7fffec34f3c0 Options.bloom_locality: 0 -2018/06/26-20:25:41.827177 7fffec34f3c0 Options.max_successive_merges: 0 -2018/06/26-20:25:41.827178 7fffec34f3c0 Options.optimize_filters_for_hits: 0 -2018/06/26-20:25:41.827179 7fffec34f3c0 Options.paranoid_file_checks: 0 -2018/06/26-20:25:41.827180 7fffec34f3c0 Options.force_consistency_checks: 0 -2018/06/26-20:25:41.827181 7fffec34f3c0 Options.report_bg_io_stats: 0 -2018/06/26-20:25:41.827677 7fffec34f3c0 [db/version_set.cc:3316] Recovered from manifest file:db-stage-3/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0 -2018/06/26-20:25:41.827683 7fffec34f3c0 [db/version_set.cc:3324] Column family [default] (ID 0), log number is 0 -2018/06/26-20:25:41.830491 7fffec34f3c0 [db/db_impl_open.cc:1188] DB pointer 0x7fbba710bc00 -2018/06/26-20:25:41.830669 7fffec34f3c0 [db/db_impl.cc:252] Shutdown: canceling all background work -2018/06/26-20:25:41.830918 7fffec34f3c0 [db/db_impl.cc:395] Shutdown complete diff --git a/testdata/db-stage-3/MANIFEST-000005 b/testdata/db-stage-3/MANIFEST-000005 deleted file mode 100644 index a3d0dfa202a..00000000000 Binary files a/testdata/db-stage-3/MANIFEST-000005 and /dev/null differ diff --git a/testdata/db-stage-3/OPTIONS-000005 b/testdata/db-stage-3/OPTIONS-000005 deleted file mode 100644 index b8b2f6b4d65..00000000000 --- a/testdata/db-stage-3/OPTIONS-000005 +++ /dev/null @@ -1,144 +0,0 @@ -# This is a RocksDB option file. -# -# For detailed file format spec, please refer to the example file -# in examples/rocksdb_option_file_example.ini -# - -[Version] - rocksdb_version=5.13.4 - options_file_version=1.1 - -[DBOptions] - allow_mmap_writes=false - base_background_compactions=-1 - new_table_reader_for_compaction_inputs=false - db_log_dir= - wal_recovery_mode=kPointInTimeRecovery - use_direct_reads=false - write_thread_max_yield_usec=100 - max_manifest_file_size=18446744073709551615 - allow_2pc=false - allow_fallocate=true - fail_if_options_file_error=false - allow_ingest_behind=false - allow_mmap_reads=false - skip_log_error_on_recovery=false - recycle_log_file_num=0 - delete_obsolete_files_period_micros=21600000000 - compaction_readahead_size=0 - use_direct_io_for_flush_and_compaction=false - log_file_time_to_roll=0 - create_missing_column_families=false - advise_random_on_open=true - max_log_file_size=0 - stats_dump_period_sec=600 - enable_thread_tracking=false - use_adaptive_mutex=false - create_if_missing=true - is_fd_close_on_exec=true - max_background_flushes=-1 - manifest_preallocation_size=4194304 - error_if_exists=true - skip_stats_update_on_db_open=false - max_open_files=-1 - random_access_max_buffer_size=1048576 - use_fsync=false - max_background_jobs=2 - two_write_queues=false - max_background_compactions=-1 - max_file_opening_threads=16 - table_cache_numshardbits=6 - keep_log_file_num=1000 - avoid_flush_during_shutdown=false - db_write_buffer_size=0 - max_total_wal_size=0 - wal_dir=db-stage-3 - max_subcompactions=1 - WAL_size_limit_MB=0 - paranoid_checks=true - allow_concurrent_memtable_write=true - writable_file_max_buffer_size=1048576 - WAL_ttl_seconds=0 - delayed_write_rate=16777216 - bytes_per_sync=0 - wal_bytes_per_sync=0 - enable_pipelined_write=false - enable_write_thread_adaptive_yield=true - write_thread_slow_yield_usec=3 - access_hint_on_compaction_start=NORMAL - info_log_level=INFO_LEVEL - dump_malloc_stats=false - avoid_flush_during_recovery=false - preserve_deletes=false - manual_wal_flush=false - - -[CFOptions "default"] - report_bg_io_stats=false - inplace_update_support=false - max_compaction_bytes=1677721600 - disable_auto_compactions=false - write_buffer_size=67108864 - bloom_locality=0 - max_bytes_for_level_multiplier=10.000000 - compaction_filter_factory=nullptr - optimize_filters_for_hits=false - target_file_size_base=67108864 - max_write_buffer_number_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - paranoid_file_checks=false - memtable_prefix_bloom_size_ratio=0.000000 - force_consistency_checks=false - max_write_buffer_number=2 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - level0_slowdown_writes_trigger=20 - level_compaction_dynamic_level_bytes=false - compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;} - inplace_update_num_locks=10000 - level0_file_num_compaction_trigger=4 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - num_levels=7 - table_factory=BlockBasedTable - compression_per_level= - target_file_size_multiplier=1 - min_write_buffer_number_to_merge=1 - arena_block_size=8388608 - max_successive_merges=0 - memtable_huge_page_size=0 - compaction_pri=kByCompensatedSize - soft_pending_compaction_bytes_limit=68719476736 - max_bytes_for_level_base=268435456 - comparator=leveldb.BytewiseComparator - max_sequential_skip_in_iterations=8 - bottommost_compression=kDisableCompressionOption - prefix_extractor=nullptr - memtable_insert_with_hint_prefix_extractor=nullptr - memtable_factory=SkipListFactory - compaction_filter=nullptr - compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} - merge_operator=nullptr - compaction_style=kCompactionStyleLevel - -[TableOptions/BlockBasedTable "default"] - format_version=2 - whole_key_filtering=true - verify_compression=false - partition_filters=false - index_block_restart_interval=1 - block_size_deviation=10 - block_size=4096 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - filter_policy=nullptr - metadata_block_size=4096 - no_block_cache=false - checksum=kCRC32c - read_amp_bytes_per_bit=8589934592 - cache_index_and_filter_blocks=false - enable_index_compression=true - index_type=kBinarySearch - hash_index_allow_collision=true - cache_index_and_filter_blocks_with_high_priority=false - flush_block_policy_factory=FlushBlockBySizePolicyFactory - diff --git a/testdata/db-stage-3/OPTIONS-000008 b/testdata/db-stage-3/OPTIONS-000008 deleted file mode 100644 index fbe0a4c8c28..00000000000 --- a/testdata/db-stage-3/OPTIONS-000008 +++ /dev/null @@ -1,144 +0,0 @@ -# This is a RocksDB option file. -# -# For detailed file format spec, please refer to the example file -# in examples/rocksdb_option_file_example.ini -# - -[Version] - rocksdb_version=5.13.4 - options_file_version=1.1 - -[DBOptions] - allow_mmap_writes=false - base_background_compactions=-1 - new_table_reader_for_compaction_inputs=false - db_log_dir= - wal_recovery_mode=kPointInTimeRecovery - use_direct_reads=false - write_thread_max_yield_usec=100 - max_manifest_file_size=18446744073709551615 - allow_2pc=false - allow_fallocate=true - fail_if_options_file_error=false - allow_ingest_behind=false - allow_mmap_reads=false - skip_log_error_on_recovery=false - recycle_log_file_num=0 - delete_obsolete_files_period_micros=21600000000 - compaction_readahead_size=0 - use_direct_io_for_flush_and_compaction=false - log_file_time_to_roll=0 - create_missing_column_families=false - advise_random_on_open=true - max_log_file_size=0 - stats_dump_period_sec=600 - enable_thread_tracking=false - use_adaptive_mutex=false - create_if_missing=false - is_fd_close_on_exec=true - max_background_flushes=-1 - manifest_preallocation_size=4194304 - error_if_exists=false - skip_stats_update_on_db_open=false - max_open_files=-1 - random_access_max_buffer_size=1048576 - use_fsync=false - max_background_jobs=2 - two_write_queues=false - max_background_compactions=-1 - max_file_opening_threads=16 - table_cache_numshardbits=6 - keep_log_file_num=1000 - avoid_flush_during_shutdown=false - db_write_buffer_size=0 - max_total_wal_size=0 - wal_dir=db-stage-3 - max_subcompactions=1 - WAL_size_limit_MB=0 - paranoid_checks=true - allow_concurrent_memtable_write=true - writable_file_max_buffer_size=1048576 - WAL_ttl_seconds=0 - delayed_write_rate=16777216 - bytes_per_sync=0 - wal_bytes_per_sync=0 - enable_pipelined_write=false - enable_write_thread_adaptive_yield=true - write_thread_slow_yield_usec=3 - access_hint_on_compaction_start=NORMAL - info_log_level=INFO_LEVEL - dump_malloc_stats=false - avoid_flush_during_recovery=false - preserve_deletes=false - manual_wal_flush=false - - -[CFOptions "default"] - report_bg_io_stats=false - inplace_update_support=false - max_compaction_bytes=1677721600 - disable_auto_compactions=false - write_buffer_size=67108864 - bloom_locality=0 - max_bytes_for_level_multiplier=10.000000 - compaction_filter_factory=nullptr - optimize_filters_for_hits=false - target_file_size_base=67108864 - max_write_buffer_number_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - paranoid_file_checks=false - memtable_prefix_bloom_size_ratio=0.000000 - force_consistency_checks=false - max_write_buffer_number=2 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - level0_slowdown_writes_trigger=20 - level_compaction_dynamic_level_bytes=false - compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;} - inplace_update_num_locks=10000 - level0_file_num_compaction_trigger=4 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - num_levels=7 - table_factory=BlockBasedTable - compression_per_level= - target_file_size_multiplier=1 - min_write_buffer_number_to_merge=1 - arena_block_size=8388608 - max_successive_merges=0 - memtable_huge_page_size=0 - compaction_pri=kByCompensatedSize - soft_pending_compaction_bytes_limit=68719476736 - max_bytes_for_level_base=268435456 - comparator=leveldb.BytewiseComparator - max_sequential_skip_in_iterations=8 - bottommost_compression=kDisableCompressionOption - prefix_extractor=nullptr - memtable_insert_with_hint_prefix_extractor=nullptr - memtable_factory=SkipListFactory - compaction_filter=nullptr - compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} - merge_operator=nullptr - compaction_style=kCompactionStyleLevel - -[TableOptions/BlockBasedTable "default"] - format_version=2 - whole_key_filtering=true - verify_compression=false - partition_filters=false - index_block_restart_interval=1 - block_size_deviation=10 - block_size=4096 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - filter_policy=nullptr - metadata_block_size=4096 - no_block_cache=false - checksum=kCRC32c - read_amp_bytes_per_bit=8589934592 - cache_index_and_filter_blocks=false - enable_index_compression=true - index_type=kBinarySearch - hash_index_allow_collision=true - cache_index_and_filter_blocks_with_high_priority=false - flush_block_policy_factory=FlushBlockBySizePolicyFactory - diff --git a/testdata/db-stage-4/000004.sst b/testdata/db-stage-4/000004.sst deleted file mode 100644 index c6c2645917c..00000000000 Binary files a/testdata/db-stage-4/000004.sst and /dev/null differ diff --git a/testdata/db-stage-4/000006.log b/testdata/db-stage-4/000006.log deleted file mode 100644 index 28e909c93c3..00000000000 Binary files a/testdata/db-stage-4/000006.log and /dev/null differ diff --git a/testdata/db-stage-4/CURRENT b/testdata/db-stage-4/CURRENT deleted file mode 100644 index aa5bb8ea509..00000000000 --- a/testdata/db-stage-4/CURRENT +++ /dev/null @@ -1 +0,0 @@ -MANIFEST-000005 diff --git a/testdata/db-stage-4/IDENTITY b/testdata/db-stage-4/IDENTITY deleted file mode 100644 index 2f69583a0ca..00000000000 --- a/testdata/db-stage-4/IDENTITY +++ /dev/null @@ -1 +0,0 @@ -153bdcaed4580998-eed9a14937098f89 \ No newline at end of file diff --git a/testdata/db-stage-4/LOCK b/testdata/db-stage-4/LOCK deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testdata/db-stage-4/LOG b/testdata/db-stage-4/LOG deleted file mode 100644 index a84681472da..00000000000 --- a/testdata/db-stage-4/LOG +++ /dev/null @@ -1,185 +0,0 @@ -2018/06/26-20:25:41.872483 7fffec34f3c0 RocksDB version: 5.13.4 -2018/06/26-20:25:41.872511 7fffec34f3c0 Git sha rocksdb_build_git_sha: -2018/06/26-20:25:41.872514 7fffec34f3c0 Compile date Jun 19 2018 -2018/06/26-20:25:41.872516 7fffec34f3c0 DB SUMMARY -2018/06/26-20:25:41.872573 7fffec34f3c0 CURRENT file: CURRENT -2018/06/26-20:25:41.872576 7fffec34f3c0 IDENTITY file: IDENTITY -2018/06/26-20:25:41.872588 7fffec34f3c0 MANIFEST file: MANIFEST-000001 size: 13 Bytes -2018/06/26-20:25:41.872591 7fffec34f3c0 SST files in db-stage-4 dir, Total Num: 0, files: -2018/06/26-20:25:41.872594 7fffec34f3c0 Write Ahead Log file in db-stage-4: 000003.log size: 139 ; -2018/06/26-20:25:41.872596 7fffec34f3c0 Options.error_if_exists: 0 -2018/06/26-20:25:41.872598 7fffec34f3c0 Options.create_if_missing: 0 -2018/06/26-20:25:41.872599 7fffec34f3c0 Options.paranoid_checks: 1 -2018/06/26-20:25:41.872600 7fffec34f3c0 Options.env: 0x10dcc2d28 -2018/06/26-20:25:41.872601 7fffec34f3c0 Options.info_log: 0x7ff2ed5000e0 -2018/06/26-20:25:41.872602 7fffec34f3c0 Options.max_file_opening_threads: 16 -2018/06/26-20:25:41.872604 7fffec34f3c0 Options.statistics: 0x0 -2018/06/26-20:25:41.872605 7fffec34f3c0 Options.use_fsync: 0 -2018/06/26-20:25:41.872606 7fffec34f3c0 Options.max_log_file_size: 0 -2018/06/26-20:25:41.872608 7fffec34f3c0 Options.max_manifest_file_size: 18446744073709551615 -2018/06/26-20:25:41.872610 7fffec34f3c0 Options.log_file_time_to_roll: 0 -2018/06/26-20:25:41.872612 7fffec34f3c0 Options.keep_log_file_num: 1000 -2018/06/26-20:25:41.872614 7fffec34f3c0 Options.recycle_log_file_num: 0 -2018/06/26-20:25:41.872615 7fffec34f3c0 Options.allow_fallocate: 1 -2018/06/26-20:25:41.872617 7fffec34f3c0 Options.allow_mmap_reads: 0 -2018/06/26-20:25:41.872618 7fffec34f3c0 Options.allow_mmap_writes: 0 -2018/06/26-20:25:41.872620 7fffec34f3c0 Options.use_direct_reads: 0 -2018/06/26-20:25:41.872622 7fffec34f3c0 Options.use_direct_io_for_flush_and_compaction: 0 -2018/06/26-20:25:41.872624 7fffec34f3c0 Options.create_missing_column_families: 0 -2018/06/26-20:25:41.872626 7fffec34f3c0 Options.db_log_dir: -2018/06/26-20:25:41.872628 7fffec34f3c0 Options.wal_dir: db-stage-4 -2018/06/26-20:25:41.872629 7fffec34f3c0 Options.table_cache_numshardbits: 6 -2018/06/26-20:25:41.872631 7fffec34f3c0 Options.max_subcompactions: 1 -2018/06/26-20:25:41.872633 7fffec34f3c0 Options.max_background_flushes: -1 -2018/06/26-20:25:41.872634 7fffec34f3c0 Options.WAL_ttl_seconds: 0 -2018/06/26-20:25:41.872636 7fffec34f3c0 Options.WAL_size_limit_MB: 0 -2018/06/26-20:25:41.872637 7fffec34f3c0 Options.manifest_preallocation_size: 4194304 -2018/06/26-20:25:41.872638 7fffec34f3c0 Options.is_fd_close_on_exec: 1 -2018/06/26-20:25:41.872639 7fffec34f3c0 Options.advise_random_on_open: 1 -2018/06/26-20:25:41.872640 7fffec34f3c0 Options.db_write_buffer_size: 0 -2018/06/26-20:25:41.872641 7fffec34f3c0 Options.write_buffer_manager: 0x7ff2ed5000b0 -2018/06/26-20:25:41.872642 7fffec34f3c0 Options.access_hint_on_compaction_start: 1 -2018/06/26-20:25:41.872643 7fffec34f3c0 Options.new_table_reader_for_compaction_inputs: 0 -2018/06/26-20:25:41.872644 7fffec34f3c0 Options.random_access_max_buffer_size: 1048576 -2018/06/26-20:25:41.872646 7fffec34f3c0 Options.use_adaptive_mutex: 0 -2018/06/26-20:25:41.872648 7fffec34f3c0 Options.rate_limiter: 0x0 -2018/06/26-20:25:41.872650 7fffec34f3c0 Options.sst_file_manager.rate_bytes_per_sec: 0 -2018/06/26-20:25:41.872652 7fffec34f3c0 Options.wal_recovery_mode: 2 -2018/06/26-20:25:41.872667 7fffec34f3c0 Options.enable_thread_tracking: 0 -2018/06/26-20:25:41.872670 7fffec34f3c0 Options.enable_pipelined_write: 0 -2018/06/26-20:25:41.872672 7fffec34f3c0 Options.allow_concurrent_memtable_write: 1 -2018/06/26-20:25:41.872673 7fffec34f3c0 Options.enable_write_thread_adaptive_yield: 1 -2018/06/26-20:25:41.872674 7fffec34f3c0 Options.write_thread_max_yield_usec: 100 -2018/06/26-20:25:41.872676 7fffec34f3c0 Options.write_thread_slow_yield_usec: 3 -2018/06/26-20:25:41.872677 7fffec34f3c0 Options.row_cache: None -2018/06/26-20:25:41.872679 7fffec34f3c0 Options.wal_filter: None -2018/06/26-20:25:41.872681 7fffec34f3c0 Options.avoid_flush_during_recovery: 0 -2018/06/26-20:25:41.872683 7fffec34f3c0 Options.allow_ingest_behind: 0 -2018/06/26-20:25:41.872684 7fffec34f3c0 Options.preserve_deletes: 0 -2018/06/26-20:25:41.872686 7fffec34f3c0 Options.two_write_queues: 0 -2018/06/26-20:25:41.872688 7fffec34f3c0 Options.manual_wal_flush: 0 -2018/06/26-20:25:41.872689 7fffec34f3c0 Options.max_background_jobs: 2 -2018/06/26-20:25:41.872691 7fffec34f3c0 Options.max_background_compactions: -1 -2018/06/26-20:25:41.872692 7fffec34f3c0 Options.avoid_flush_during_shutdown: 0 -2018/06/26-20:25:41.872694 7fffec34f3c0 Options.writable_file_max_buffer_size: 1048576 -2018/06/26-20:25:41.872696 7fffec34f3c0 Options.delayed_write_rate : 16777216 -2018/06/26-20:25:41.872697 7fffec34f3c0 Options.max_total_wal_size: 0 -2018/06/26-20:25:41.872699 7fffec34f3c0 Options.delete_obsolete_files_period_micros: 21600000000 -2018/06/26-20:25:41.872701 7fffec34f3c0 Options.stats_dump_period_sec: 600 -2018/06/26-20:25:41.872702 7fffec34f3c0 Options.max_open_files: -1 -2018/06/26-20:25:41.872704 7fffec34f3c0 Options.bytes_per_sync: 0 -2018/06/26-20:25:41.872706 7fffec34f3c0 Options.wal_bytes_per_sync: 0 -2018/06/26-20:25:41.872708 7fffec34f3c0 Options.compaction_readahead_size: 0 -2018/06/26-20:25:41.872709 7fffec34f3c0 Compression algorithms supported: -2018/06/26-20:25:41.872711 7fffec34f3c0 kZSTD supported: 0 -2018/06/26-20:25:41.872713 7fffec34f3c0 kZlibCompression supported: 1 -2018/06/26-20:25:41.872731 7fffec34f3c0 kXpressCompression supported: 0 -2018/06/26-20:25:41.872734 7fffec34f3c0 kSnappyCompression supported: 1 -2018/06/26-20:25:41.872736 7fffec34f3c0 kZSTDNotFinalCompression supported: 0 -2018/06/26-20:25:41.872737 7fffec34f3c0 kLZ4HCCompression supported: 1 -2018/06/26-20:25:41.872738 7fffec34f3c0 kLZ4Compression supported: 1 -2018/06/26-20:25:41.872739 7fffec34f3c0 kBZip2Compression supported: 1 -2018/06/26-20:25:41.872741 7fffec34f3c0 Fast CRC32 supported: Not supported on x86 -2018/06/26-20:25:41.872828 7fffec34f3c0 [db/version_set.cc:3050] Recovering from manifest file: MANIFEST-000001 -2018/06/26-20:25:41.872855 7fffec34f3c0 [db/column_family.cc:433] --------------- Options for column family [default]: -2018/06/26-20:25:41.872857 7fffec34f3c0 Options.comparator: leveldb.BytewiseComparator -2018/06/26-20:25:41.872858 7fffec34f3c0 Options.merge_operator: None -2018/06/26-20:25:41.872859 7fffec34f3c0 Options.compaction_filter: None -2018/06/26-20:25:41.872861 7fffec34f3c0 Options.compaction_filter_factory: None -2018/06/26-20:25:41.872862 7fffec34f3c0 Options.memtable_factory: SkipListFactory -2018/06/26-20:25:41.872863 7fffec34f3c0 Options.table_factory: BlockBasedTable -2018/06/26-20:25:41.872883 7fffec34f3c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7ff2ed63daa0) - cache_index_and_filter_blocks: 0 - cache_index_and_filter_blocks_with_high_priority: 0 - pin_l0_filter_and_index_blocks_in_cache: 0 - index_type: 0 - hash_index_allow_collision: 1 - checksum: 1 - no_block_cache: 0 - block_cache: 0x7ff2ed661d48 - block_cache_name: LRUCache - block_cache_options: - capacity : 8388608 - num_shard_bits : 4 - strict_capacity_limit : 0 - high_pri_pool_ratio: 0.000 - block_cache_compressed: 0x0 - persistent_cache: 0x0 - block_size: 4096 - block_size_deviation: 10 - block_restart_interval: 16 - index_block_restart_interval: 1 - metadata_block_size: 4096 - partition_filters: 0 - use_delta_encoding: 1 - filter_policy: nullptr - whole_key_filtering: 1 - verify_compression: 0 - read_amp_bytes_per_bit: 0 - format_version: 2 - enable_index_compression: 1 -2018/06/26-20:25:41.872900 7fffec34f3c0 Options.write_buffer_size: 67108864 -2018/06/26-20:25:41.872902 7fffec34f3c0 Options.max_write_buffer_number: 2 -2018/06/26-20:25:41.872904 7fffec34f3c0 Options.compression: Snappy -2018/06/26-20:25:41.872905 7fffec34f3c0 Options.bottommost_compression: Disabled -2018/06/26-20:25:41.872906 7fffec34f3c0 Options.prefix_extractor: nullptr -2018/06/26-20:25:41.872907 7fffec34f3c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr -2018/06/26-20:25:41.872908 7fffec34f3c0 Options.num_levels: 7 -2018/06/26-20:25:41.872909 7fffec34f3c0 Options.min_write_buffer_number_to_merge: 1 -2018/06/26-20:25:41.872911 7fffec34f3c0 Options.max_write_buffer_number_to_maintain: 0 -2018/06/26-20:25:41.872912 7fffec34f3c0 Options.compression_opts.window_bits: -14 -2018/06/26-20:25:41.872913 7fffec34f3c0 Options.compression_opts.level: 32767 -2018/06/26-20:25:41.872914 7fffec34f3c0 Options.compression_opts.strategy: 0 -2018/06/26-20:25:41.872915 7fffec34f3c0 Options.compression_opts.max_dict_bytes: 0 -2018/06/26-20:25:41.872916 7fffec34f3c0 Options.level0_file_num_compaction_trigger: 4 -2018/06/26-20:25:41.872917 7fffec34f3c0 Options.level0_slowdown_writes_trigger: 20 -2018/06/26-20:25:41.872918 7fffec34f3c0 Options.level0_stop_writes_trigger: 36 -2018/06/26-20:25:41.872920 7fffec34f3c0 Options.target_file_size_base: 67108864 -2018/06/26-20:25:41.872921 7fffec34f3c0 Options.target_file_size_multiplier: 1 -2018/06/26-20:25:41.872922 7fffec34f3c0 Options.max_bytes_for_level_base: 268435456 -2018/06/26-20:25:41.872923 7fffec34f3c0 Options.level_compaction_dynamic_level_bytes: 0 -2018/06/26-20:25:41.872924 7fffec34f3c0 Options.max_bytes_for_level_multiplier: 10.000000 -2018/06/26-20:25:41.872926 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 -2018/06/26-20:25:41.872928 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 -2018/06/26-20:25:41.872929 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 -2018/06/26-20:25:41.872930 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 -2018/06/26-20:25:41.872931 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 -2018/06/26-20:25:41.872932 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 -2018/06/26-20:25:41.872933 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 -2018/06/26-20:25:41.872934 7fffec34f3c0 Options.max_sequential_skip_in_iterations: 8 -2018/06/26-20:25:41.872935 7fffec34f3c0 Options.max_compaction_bytes: 1677721600 -2018/06/26-20:25:41.872937 7fffec34f3c0 Options.arena_block_size: 8388608 -2018/06/26-20:25:41.872938 7fffec34f3c0 Options.soft_pending_compaction_bytes_limit: 68719476736 -2018/06/26-20:25:41.872939 7fffec34f3c0 Options.hard_pending_compaction_bytes_limit: 274877906944 -2018/06/26-20:25:41.872940 7fffec34f3c0 Options.rate_limit_delay_max_milliseconds: 100 -2018/06/26-20:25:41.872941 7fffec34f3c0 Options.disable_auto_compactions: 0 -2018/06/26-20:25:41.872943 7fffec34f3c0 Options.compaction_style: kCompactionStyleLevel -2018/06/26-20:25:41.872944 7fffec34f3c0 Options.compaction_pri: kByCompensatedSize -2018/06/26-20:25:41.872945 7fffec34f3c0 Options.compaction_options_universal.size_ratio: 1 -2018/06/26-20:25:41.872954 7fffec34f3c0 Options.compaction_options_universal.min_merge_width: 2 -2018/06/26-20:25:41.872956 7fffec34f3c0 Options.compaction_options_universal.max_merge_width: 4294967295 -2018/06/26-20:25:41.872957 7fffec34f3c0 Options.compaction_options_universal.max_size_amplification_percent: 200 -2018/06/26-20:25:41.872959 7fffec34f3c0 Options.compaction_options_universal.compression_size_percent: -1 -2018/06/26-20:25:41.872960 7fffec34f3c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize -2018/06/26-20:25:41.872962 7fffec34f3c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 -2018/06/26-20:25:41.872963 7fffec34f3c0 Options.compaction_options_fifo.allow_compaction: 0 -2018/06/26-20:25:41.872964 7fffec34f3c0 Options.compaction_options_fifo.ttl: 0 -2018/06/26-20:25:41.872965 7fffec34f3c0 Options.table_properties_collectors: -2018/06/26-20:25:41.872966 7fffec34f3c0 Options.inplace_update_support: 0 -2018/06/26-20:25:41.872967 7fffec34f3c0 Options.inplace_update_num_locks: 10000 -2018/06/26-20:25:41.872969 7fffec34f3c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 -2018/06/26-20:25:41.872970 7fffec34f3c0 Options.memtable_huge_page_size: 0 -2018/06/26-20:25:41.872971 7fffec34f3c0 Options.bloom_locality: 0 -2018/06/26-20:25:41.872972 7fffec34f3c0 Options.max_successive_merges: 0 -2018/06/26-20:25:41.872973 7fffec34f3c0 Options.optimize_filters_for_hits: 0 -2018/06/26-20:25:41.872974 7fffec34f3c0 Options.paranoid_file_checks: 0 -2018/06/26-20:25:41.872975 7fffec34f3c0 Options.force_consistency_checks: 0 -2018/06/26-20:25:41.872976 7fffec34f3c0 Options.report_bg_io_stats: 0 -2018/06/26-20:25:41.873412 7fffec34f3c0 [db/version_set.cc:3316] Recovered from manifest file:db-stage-4/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0 -2018/06/26-20:25:41.873418 7fffec34f3c0 [db/version_set.cc:3324] Column family [default] (ID 0), log number is 0 -2018/06/26-20:25:41.873494 7fffec34f3c0 EVENT_LOG_v1 {"time_micros": 1530059141873481, "job": 1, "event": "recovery_started", "log_files": [3]} -2018/06/26-20:25:41.873500 7fffec34f3c0 [db/db_impl_open.cc:541] Recovering log #3 mode 2 -2018/06/26-20:25:41.874247 7fffec34f3c0 EVENT_LOG_v1 {"time_micros": 1530059141874229, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 986, "table_properties": {"data_size": 62, "index_size": 27, "filter_size": 0, "raw_key_size": 33, "raw_average_key_size": 11, "raw_value_size": 9, "raw_average_value_size": 3, "num_data_blocks": 1, "num_entries": 3, "filter_policy_name": "", "kDeletedKeys": "1", "kMergeOperands": "0"}} -2018/06/26-20:25:41.874310 7fffec34f3c0 [db/version_set.cc:2825] Creating manifest 5 -2018/06/26-20:25:41.875205 7fffec34f3c0 EVENT_LOG_v1 {"time_micros": 1530059141875198, "job": 1, "event": "recovery_finished"} -2018/06/26-20:25:41.880183 7fffec34f3c0 [db/db_impl_open.cc:1188] DB pointer 0x7ff2ee001200 diff --git a/testdata/db-stage-4/LOG.old.1530059141872089 b/testdata/db-stage-4/LOG.old.1530059141872089 deleted file mode 100644 index 87e6279177f..00000000000 --- a/testdata/db-stage-4/LOG.old.1530059141872089 +++ /dev/null @@ -1,180 +0,0 @@ -2018/06/26-20:25:41.865657 7fffec34f3c0 RocksDB version: 5.13.4 -2018/06/26-20:25:41.866228 7fffec34f3c0 Git sha rocksdb_build_git_sha: -2018/06/26-20:25:41.866231 7fffec34f3c0 Compile date Jun 19 2018 -2018/06/26-20:25:41.866233 7fffec34f3c0 DB SUMMARY -2018/06/26-20:25:41.866275 7fffec34f3c0 SST files in db-stage-4 dir, Total Num: 0, files: -2018/06/26-20:25:41.866278 7fffec34f3c0 Write Ahead Log file in db-stage-4: -2018/06/26-20:25:41.866280 7fffec34f3c0 Options.error_if_exists: 1 -2018/06/26-20:25:41.866284 7fffec34f3c0 Options.create_if_missing: 1 -2018/06/26-20:25:41.866285 7fffec34f3c0 Options.paranoid_checks: 1 -2018/06/26-20:25:41.866286 7fffec34f3c0 Options.env: 0x10dcc2d28 -2018/06/26-20:25:41.866288 7fffec34f3c0 Options.info_log: 0x7ff2ed6634f0 -2018/06/26-20:25:41.866289 7fffec34f3c0 Options.max_file_opening_threads: 16 -2018/06/26-20:25:41.866290 7fffec34f3c0 Options.statistics: 0x0 -2018/06/26-20:25:41.866291 7fffec34f3c0 Options.use_fsync: 0 -2018/06/26-20:25:41.866292 7fffec34f3c0 Options.max_log_file_size: 0 -2018/06/26-20:25:41.866293 7fffec34f3c0 Options.max_manifest_file_size: 18446744073709551615 -2018/06/26-20:25:41.866294 7fffec34f3c0 Options.log_file_time_to_roll: 0 -2018/06/26-20:25:41.866295 7fffec34f3c0 Options.keep_log_file_num: 1000 -2018/06/26-20:25:41.866296 7fffec34f3c0 Options.recycle_log_file_num: 0 -2018/06/26-20:25:41.866297 7fffec34f3c0 Options.allow_fallocate: 1 -2018/06/26-20:25:41.866298 7fffec34f3c0 Options.allow_mmap_reads: 0 -2018/06/26-20:25:41.866299 7fffec34f3c0 Options.allow_mmap_writes: 0 -2018/06/26-20:25:41.866301 7fffec34f3c0 Options.use_direct_reads: 0 -2018/06/26-20:25:41.866302 7fffec34f3c0 Options.use_direct_io_for_flush_and_compaction: 0 -2018/06/26-20:25:41.866303 7fffec34f3c0 Options.create_missing_column_families: 0 -2018/06/26-20:25:41.866304 7fffec34f3c0 Options.db_log_dir: -2018/06/26-20:25:41.866305 7fffec34f3c0 Options.wal_dir: db-stage-4 -2018/06/26-20:25:41.866306 7fffec34f3c0 Options.table_cache_numshardbits: 6 -2018/06/26-20:25:41.866307 7fffec34f3c0 Options.max_subcompactions: 1 -2018/06/26-20:25:41.866308 7fffec34f3c0 Options.max_background_flushes: -1 -2018/06/26-20:25:41.866309 7fffec34f3c0 Options.WAL_ttl_seconds: 0 -2018/06/26-20:25:41.866310 7fffec34f3c0 Options.WAL_size_limit_MB: 0 -2018/06/26-20:25:41.866311 7fffec34f3c0 Options.manifest_preallocation_size: 4194304 -2018/06/26-20:25:41.866312 7fffec34f3c0 Options.is_fd_close_on_exec: 1 -2018/06/26-20:25:41.866313 7fffec34f3c0 Options.advise_random_on_open: 1 -2018/06/26-20:25:41.866314 7fffec34f3c0 Options.db_write_buffer_size: 0 -2018/06/26-20:25:41.866316 7fffec34f3c0 Options.write_buffer_manager: 0x7ff2ed6634c0 -2018/06/26-20:25:41.866317 7fffec34f3c0 Options.access_hint_on_compaction_start: 1 -2018/06/26-20:25:41.866318 7fffec34f3c0 Options.new_table_reader_for_compaction_inputs: 0 -2018/06/26-20:25:41.866319 7fffec34f3c0 Options.random_access_max_buffer_size: 1048576 -2018/06/26-20:25:41.866320 7fffec34f3c0 Options.use_adaptive_mutex: 0 -2018/06/26-20:25:41.866321 7fffec34f3c0 Options.rate_limiter: 0x0 -2018/06/26-20:25:41.866322 7fffec34f3c0 Options.sst_file_manager.rate_bytes_per_sec: 0 -2018/06/26-20:25:41.866323 7fffec34f3c0 Options.wal_recovery_mode: 2 -2018/06/26-20:25:41.866324 7fffec34f3c0 Options.enable_thread_tracking: 0 -2018/06/26-20:25:41.866325 7fffec34f3c0 Options.enable_pipelined_write: 0 -2018/06/26-20:25:41.866393 7fffec34f3c0 Options.allow_concurrent_memtable_write: 1 -2018/06/26-20:25:41.866395 7fffec34f3c0 Options.enable_write_thread_adaptive_yield: 1 -2018/06/26-20:25:41.866396 7fffec34f3c0 Options.write_thread_max_yield_usec: 100 -2018/06/26-20:25:41.866398 7fffec34f3c0 Options.write_thread_slow_yield_usec: 3 -2018/06/26-20:25:41.866399 7fffec34f3c0 Options.row_cache: None -2018/06/26-20:25:41.866400 7fffec34f3c0 Options.wal_filter: None -2018/06/26-20:25:41.866401 7fffec34f3c0 Options.avoid_flush_during_recovery: 0 -2018/06/26-20:25:41.866402 7fffec34f3c0 Options.allow_ingest_behind: 0 -2018/06/26-20:25:41.866406 7fffec34f3c0 Options.preserve_deletes: 0 -2018/06/26-20:25:41.866407 7fffec34f3c0 Options.two_write_queues: 0 -2018/06/26-20:25:41.866408 7fffec34f3c0 Options.manual_wal_flush: 0 -2018/06/26-20:25:41.866409 7fffec34f3c0 Options.max_background_jobs: 2 -2018/06/26-20:25:41.866410 7fffec34f3c0 Options.max_background_compactions: -1 -2018/06/26-20:25:41.866411 7fffec34f3c0 Options.avoid_flush_during_shutdown: 0 -2018/06/26-20:25:41.866412 7fffec34f3c0 Options.writable_file_max_buffer_size: 1048576 -2018/06/26-20:25:41.866413 7fffec34f3c0 Options.delayed_write_rate : 16777216 -2018/06/26-20:25:41.866415 7fffec34f3c0 Options.max_total_wal_size: 0 -2018/06/26-20:25:41.866416 7fffec34f3c0 Options.delete_obsolete_files_period_micros: 21600000000 -2018/06/26-20:25:41.866417 7fffec34f3c0 Options.stats_dump_period_sec: 600 -2018/06/26-20:25:41.866418 7fffec34f3c0 Options.max_open_files: -1 -2018/06/26-20:25:41.866419 7fffec34f3c0 Options.bytes_per_sync: 0 -2018/06/26-20:25:41.866420 7fffec34f3c0 Options.wal_bytes_per_sync: 0 -2018/06/26-20:25:41.866421 7fffec34f3c0 Options.compaction_readahead_size: 0 -2018/06/26-20:25:41.866422 7fffec34f3c0 Compression algorithms supported: -2018/06/26-20:25:41.866424 7fffec34f3c0 kZSTD supported: 0 -2018/06/26-20:25:41.866425 7fffec34f3c0 kZlibCompression supported: 1 -2018/06/26-20:25:41.866426 7fffec34f3c0 kXpressCompression supported: 0 -2018/06/26-20:25:41.866427 7fffec34f3c0 kSnappyCompression supported: 1 -2018/06/26-20:25:41.866428 7fffec34f3c0 kZSTDNotFinalCompression supported: 0 -2018/06/26-20:25:41.866430 7fffec34f3c0 kLZ4HCCompression supported: 1 -2018/06/26-20:25:41.866431 7fffec34f3c0 kLZ4Compression supported: 1 -2018/06/26-20:25:41.866432 7fffec34f3c0 kBZip2Compression supported: 1 -2018/06/26-20:25:41.866436 7fffec34f3c0 Fast CRC32 supported: Not supported on x86 -2018/06/26-20:25:41.866623 7fffec34f3c0 [db/db_impl_open.cc:227] Creating manifest 1 -2018/06/26-20:25:41.867765 7fffec34f3c0 [db/version_set.cc:3050] Recovering from manifest file: MANIFEST-000001 -2018/06/26-20:25:41.867818 7fffec34f3c0 [db/column_family.cc:433] --------------- Options for column family [default]: -2018/06/26-20:25:41.867821 7fffec34f3c0 Options.comparator: leveldb.BytewiseComparator -2018/06/26-20:25:41.867822 7fffec34f3c0 Options.merge_operator: None -2018/06/26-20:25:41.867823 7fffec34f3c0 Options.compaction_filter: None -2018/06/26-20:25:41.867825 7fffec34f3c0 Options.compaction_filter_factory: None -2018/06/26-20:25:41.867826 7fffec34f3c0 Options.memtable_factory: SkipListFactory -2018/06/26-20:25:41.867827 7fffec34f3c0 Options.table_factory: BlockBasedTable -2018/06/26-20:25:41.867852 7fffec34f3c0 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7ff2ed63daa0) - cache_index_and_filter_blocks: 0 - cache_index_and_filter_blocks_with_high_priority: 0 - pin_l0_filter_and_index_blocks_in_cache: 0 - index_type: 0 - hash_index_allow_collision: 1 - checksum: 1 - no_block_cache: 0 - block_cache: 0x7ff2ed661d48 - block_cache_name: LRUCache - block_cache_options: - capacity : 8388608 - num_shard_bits : 4 - strict_capacity_limit : 0 - high_pri_pool_ratio: 0.000 - block_cache_compressed: 0x0 - persistent_cache: 0x0 - block_size: 4096 - block_size_deviation: 10 - block_restart_interval: 16 - index_block_restart_interval: 1 - metadata_block_size: 4096 - partition_filters: 0 - use_delta_encoding: 1 - filter_policy: nullptr - whole_key_filtering: 1 - verify_compression: 0 - read_amp_bytes_per_bit: 0 - format_version: 2 - enable_index_compression: 1 -2018/06/26-20:25:41.867876 7fffec34f3c0 Options.write_buffer_size: 67108864 -2018/06/26-20:25:41.867878 7fffec34f3c0 Options.max_write_buffer_number: 2 -2018/06/26-20:25:41.867880 7fffec34f3c0 Options.compression: Snappy -2018/06/26-20:25:41.867881 7fffec34f3c0 Options.bottommost_compression: Disabled -2018/06/26-20:25:41.867882 7fffec34f3c0 Options.prefix_extractor: nullptr -2018/06/26-20:25:41.867883 7fffec34f3c0 Options.memtable_insert_with_hint_prefix_extractor: nullptr -2018/06/26-20:25:41.867884 7fffec34f3c0 Options.num_levels: 7 -2018/06/26-20:25:41.867885 7fffec34f3c0 Options.min_write_buffer_number_to_merge: 1 -2018/06/26-20:25:41.867887 7fffec34f3c0 Options.max_write_buffer_number_to_maintain: 0 -2018/06/26-20:25:41.867888 7fffec34f3c0 Options.compression_opts.window_bits: -14 -2018/06/26-20:25:41.867889 7fffec34f3c0 Options.compression_opts.level: 32767 -2018/06/26-20:25:41.867890 7fffec34f3c0 Options.compression_opts.strategy: 0 -2018/06/26-20:25:41.867891 7fffec34f3c0 Options.compression_opts.max_dict_bytes: 0 -2018/06/26-20:25:41.867892 7fffec34f3c0 Options.level0_file_num_compaction_trigger: 4 -2018/06/26-20:25:41.867893 7fffec34f3c0 Options.level0_slowdown_writes_trigger: 20 -2018/06/26-20:25:41.867894 7fffec34f3c0 Options.level0_stop_writes_trigger: 36 -2018/06/26-20:25:41.867895 7fffec34f3c0 Options.target_file_size_base: 67108864 -2018/06/26-20:25:41.867897 7fffec34f3c0 Options.target_file_size_multiplier: 1 -2018/06/26-20:25:41.867898 7fffec34f3c0 Options.max_bytes_for_level_base: 268435456 -2018/06/26-20:25:41.867899 7fffec34f3c0 Options.level_compaction_dynamic_level_bytes: 0 -2018/06/26-20:25:41.867900 7fffec34f3c0 Options.max_bytes_for_level_multiplier: 10.000000 -2018/06/26-20:25:41.867902 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[0]: 1 -2018/06/26-20:25:41.867903 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[1]: 1 -2018/06/26-20:25:41.867905 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[2]: 1 -2018/06/26-20:25:41.867906 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[3]: 1 -2018/06/26-20:25:41.867907 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[4]: 1 -2018/06/26-20:25:41.867908 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[5]: 1 -2018/06/26-20:25:41.867909 7fffec34f3c0 Options.max_bytes_for_level_multiplier_addtl[6]: 1 -2018/06/26-20:25:41.867910 7fffec34f3c0 Options.max_sequential_skip_in_iterations: 8 -2018/06/26-20:25:41.867911 7fffec34f3c0 Options.max_compaction_bytes: 1677721600 -2018/06/26-20:25:41.867912 7fffec34f3c0 Options.arena_block_size: 8388608 -2018/06/26-20:25:41.867913 7fffec34f3c0 Options.soft_pending_compaction_bytes_limit: 68719476736 -2018/06/26-20:25:41.867915 7fffec34f3c0 Options.hard_pending_compaction_bytes_limit: 274877906944 -2018/06/26-20:25:41.867916 7fffec34f3c0 Options.rate_limit_delay_max_milliseconds: 100 -2018/06/26-20:25:41.867917 7fffec34f3c0 Options.disable_auto_compactions: 0 -2018/06/26-20:25:41.867919 7fffec34f3c0 Options.compaction_style: kCompactionStyleLevel -2018/06/26-20:25:41.867920 7fffec34f3c0 Options.compaction_pri: kByCompensatedSize -2018/06/26-20:25:41.867921 7fffec34f3c0 Options.compaction_options_universal.size_ratio: 1 -2018/06/26-20:25:41.867922 7fffec34f3c0 Options.compaction_options_universal.min_merge_width: 2 -2018/06/26-20:25:41.867923 7fffec34f3c0 Options.compaction_options_universal.max_merge_width: 4294967295 -2018/06/26-20:25:41.867932 7fffec34f3c0 Options.compaction_options_universal.max_size_amplification_percent: 200 -2018/06/26-20:25:41.867934 7fffec34f3c0 Options.compaction_options_universal.compression_size_percent: -1 -2018/06/26-20:25:41.867936 7fffec34f3c0 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize -2018/06/26-20:25:41.867937 7fffec34f3c0 Options.compaction_options_fifo.max_table_files_size: 1073741824 -2018/06/26-20:25:41.867938 7fffec34f3c0 Options.compaction_options_fifo.allow_compaction: 0 -2018/06/26-20:25:41.867940 7fffec34f3c0 Options.compaction_options_fifo.ttl: 0 -2018/06/26-20:25:41.867941 7fffec34f3c0 Options.table_properties_collectors: -2018/06/26-20:25:41.867942 7fffec34f3c0 Options.inplace_update_support: 0 -2018/06/26-20:25:41.867943 7fffec34f3c0 Options.inplace_update_num_locks: 10000 -2018/06/26-20:25:41.867944 7fffec34f3c0 Options.memtable_prefix_bloom_size_ratio: 0.000000 -2018/06/26-20:25:41.867945 7fffec34f3c0 Options.memtable_huge_page_size: 0 -2018/06/26-20:25:41.867947 7fffec34f3c0 Options.bloom_locality: 0 -2018/06/26-20:25:41.867948 7fffec34f3c0 Options.max_successive_merges: 0 -2018/06/26-20:25:41.867949 7fffec34f3c0 Options.optimize_filters_for_hits: 0 -2018/06/26-20:25:41.867950 7fffec34f3c0 Options.paranoid_file_checks: 0 -2018/06/26-20:25:41.867951 7fffec34f3c0 Options.force_consistency_checks: 0 -2018/06/26-20:25:41.867952 7fffec34f3c0 Options.report_bg_io_stats: 0 -2018/06/26-20:25:41.868475 7fffec34f3c0 [db/version_set.cc:3316] Recovered from manifest file:db-stage-4/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0 -2018/06/26-20:25:41.868479 7fffec34f3c0 [db/version_set.cc:3324] Column family [default] (ID 0), log number is 0 -2018/06/26-20:25:41.871627 7fffec34f3c0 [db/db_impl_open.cc:1188] DB pointer 0x7ff2ee86d600 -2018/06/26-20:25:41.871751 7fffec34f3c0 [db/db_impl.cc:252] Shutdown: canceling all background work -2018/06/26-20:25:41.871952 7fffec34f3c0 [db/db_impl.cc:395] Shutdown complete diff --git a/testdata/db-stage-4/MANIFEST-000005 b/testdata/db-stage-4/MANIFEST-000005 deleted file mode 100644 index a3d0dfa202a..00000000000 Binary files a/testdata/db-stage-4/MANIFEST-000005 and /dev/null differ diff --git a/testdata/db-stage-4/OPTIONS-000005 b/testdata/db-stage-4/OPTIONS-000005 deleted file mode 100644 index 2856bb9ef31..00000000000 --- a/testdata/db-stage-4/OPTIONS-000005 +++ /dev/null @@ -1,144 +0,0 @@ -# This is a RocksDB option file. -# -# For detailed file format spec, please refer to the example file -# in examples/rocksdb_option_file_example.ini -# - -[Version] - rocksdb_version=5.13.4 - options_file_version=1.1 - -[DBOptions] - allow_mmap_writes=false - base_background_compactions=-1 - new_table_reader_for_compaction_inputs=false - db_log_dir= - wal_recovery_mode=kPointInTimeRecovery - use_direct_reads=false - write_thread_max_yield_usec=100 - max_manifest_file_size=18446744073709551615 - allow_2pc=false - allow_fallocate=true - fail_if_options_file_error=false - allow_ingest_behind=false - allow_mmap_reads=false - skip_log_error_on_recovery=false - recycle_log_file_num=0 - delete_obsolete_files_period_micros=21600000000 - compaction_readahead_size=0 - use_direct_io_for_flush_and_compaction=false - log_file_time_to_roll=0 - create_missing_column_families=false - advise_random_on_open=true - max_log_file_size=0 - stats_dump_period_sec=600 - enable_thread_tracking=false - use_adaptive_mutex=false - create_if_missing=true - is_fd_close_on_exec=true - max_background_flushes=-1 - manifest_preallocation_size=4194304 - error_if_exists=true - skip_stats_update_on_db_open=false - max_open_files=-1 - random_access_max_buffer_size=1048576 - use_fsync=false - max_background_jobs=2 - two_write_queues=false - max_background_compactions=-1 - max_file_opening_threads=16 - table_cache_numshardbits=6 - keep_log_file_num=1000 - avoid_flush_during_shutdown=false - db_write_buffer_size=0 - max_total_wal_size=0 - wal_dir=db-stage-4 - max_subcompactions=1 - WAL_size_limit_MB=0 - paranoid_checks=true - allow_concurrent_memtable_write=true - writable_file_max_buffer_size=1048576 - WAL_ttl_seconds=0 - delayed_write_rate=16777216 - bytes_per_sync=0 - wal_bytes_per_sync=0 - enable_pipelined_write=false - enable_write_thread_adaptive_yield=true - write_thread_slow_yield_usec=3 - access_hint_on_compaction_start=NORMAL - info_log_level=INFO_LEVEL - dump_malloc_stats=false - avoid_flush_during_recovery=false - preserve_deletes=false - manual_wal_flush=false - - -[CFOptions "default"] - report_bg_io_stats=false - inplace_update_support=false - max_compaction_bytes=1677721600 - disable_auto_compactions=false - write_buffer_size=67108864 - bloom_locality=0 - max_bytes_for_level_multiplier=10.000000 - compaction_filter_factory=nullptr - optimize_filters_for_hits=false - target_file_size_base=67108864 - max_write_buffer_number_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - paranoid_file_checks=false - memtable_prefix_bloom_size_ratio=0.000000 - force_consistency_checks=false - max_write_buffer_number=2 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - level0_slowdown_writes_trigger=20 - level_compaction_dynamic_level_bytes=false - compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;} - inplace_update_num_locks=10000 - level0_file_num_compaction_trigger=4 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - num_levels=7 - table_factory=BlockBasedTable - compression_per_level= - target_file_size_multiplier=1 - min_write_buffer_number_to_merge=1 - arena_block_size=8388608 - max_successive_merges=0 - memtable_huge_page_size=0 - compaction_pri=kByCompensatedSize - soft_pending_compaction_bytes_limit=68719476736 - max_bytes_for_level_base=268435456 - comparator=leveldb.BytewiseComparator - max_sequential_skip_in_iterations=8 - bottommost_compression=kDisableCompressionOption - prefix_extractor=nullptr - memtable_insert_with_hint_prefix_extractor=nullptr - memtable_factory=SkipListFactory - compaction_filter=nullptr - compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} - merge_operator=nullptr - compaction_style=kCompactionStyleLevel - -[TableOptions/BlockBasedTable "default"] - format_version=2 - whole_key_filtering=true - verify_compression=false - partition_filters=false - index_block_restart_interval=1 - block_size_deviation=10 - block_size=4096 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - filter_policy=nullptr - metadata_block_size=4096 - no_block_cache=false - checksum=kCRC32c - read_amp_bytes_per_bit=8589934592 - cache_index_and_filter_blocks=false - enable_index_compression=true - index_type=kBinarySearch - hash_index_allow_collision=true - cache_index_and_filter_blocks_with_high_priority=false - flush_block_policy_factory=FlushBlockBySizePolicyFactory - diff --git a/testdata/db-stage-4/OPTIONS-000008 b/testdata/db-stage-4/OPTIONS-000008 deleted file mode 100644 index b13f75226e6..00000000000 --- a/testdata/db-stage-4/OPTIONS-000008 +++ /dev/null @@ -1,144 +0,0 @@ -# This is a RocksDB option file. -# -# For detailed file format spec, please refer to the example file -# in examples/rocksdb_option_file_example.ini -# - -[Version] - rocksdb_version=5.13.4 - options_file_version=1.1 - -[DBOptions] - allow_mmap_writes=false - base_background_compactions=-1 - new_table_reader_for_compaction_inputs=false - db_log_dir= - wal_recovery_mode=kPointInTimeRecovery - use_direct_reads=false - write_thread_max_yield_usec=100 - max_manifest_file_size=18446744073709551615 - allow_2pc=false - allow_fallocate=true - fail_if_options_file_error=false - allow_ingest_behind=false - allow_mmap_reads=false - skip_log_error_on_recovery=false - recycle_log_file_num=0 - delete_obsolete_files_period_micros=21600000000 - compaction_readahead_size=0 - use_direct_io_for_flush_and_compaction=false - log_file_time_to_roll=0 - create_missing_column_families=false - advise_random_on_open=true - max_log_file_size=0 - stats_dump_period_sec=600 - enable_thread_tracking=false - use_adaptive_mutex=false - create_if_missing=false - is_fd_close_on_exec=true - max_background_flushes=-1 - manifest_preallocation_size=4194304 - error_if_exists=false - skip_stats_update_on_db_open=false - max_open_files=-1 - random_access_max_buffer_size=1048576 - use_fsync=false - max_background_jobs=2 - two_write_queues=false - max_background_compactions=-1 - max_file_opening_threads=16 - table_cache_numshardbits=6 - keep_log_file_num=1000 - avoid_flush_during_shutdown=false - db_write_buffer_size=0 - max_total_wal_size=0 - wal_dir=db-stage-4 - max_subcompactions=1 - WAL_size_limit_MB=0 - paranoid_checks=true - allow_concurrent_memtable_write=true - writable_file_max_buffer_size=1048576 - WAL_ttl_seconds=0 - delayed_write_rate=16777216 - bytes_per_sync=0 - wal_bytes_per_sync=0 - enable_pipelined_write=false - enable_write_thread_adaptive_yield=true - write_thread_slow_yield_usec=3 - access_hint_on_compaction_start=NORMAL - info_log_level=INFO_LEVEL - dump_malloc_stats=false - avoid_flush_during_recovery=false - preserve_deletes=false - manual_wal_flush=false - - -[CFOptions "default"] - report_bg_io_stats=false - inplace_update_support=false - max_compaction_bytes=1677721600 - disable_auto_compactions=false - write_buffer_size=67108864 - bloom_locality=0 - max_bytes_for_level_multiplier=10.000000 - compaction_filter_factory=nullptr - optimize_filters_for_hits=false - target_file_size_base=67108864 - max_write_buffer_number_to_maintain=0 - hard_pending_compaction_bytes_limit=274877906944 - paranoid_file_checks=false - memtable_prefix_bloom_size_ratio=0.000000 - force_consistency_checks=false - max_write_buffer_number=2 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - level0_slowdown_writes_trigger=20 - level_compaction_dynamic_level_bytes=false - compaction_options_fifo={allow_compaction=false;ttl=0;max_table_files_size=1073741824;} - inplace_update_num_locks=10000 - level0_file_num_compaction_trigger=4 - compression=kSnappyCompression - level0_stop_writes_trigger=36 - num_levels=7 - table_factory=BlockBasedTable - compression_per_level= - target_file_size_multiplier=1 - min_write_buffer_number_to_merge=1 - arena_block_size=8388608 - max_successive_merges=0 - memtable_huge_page_size=0 - compaction_pri=kByCompensatedSize - soft_pending_compaction_bytes_limit=68719476736 - max_bytes_for_level_base=268435456 - comparator=leveldb.BytewiseComparator - max_sequential_skip_in_iterations=8 - bottommost_compression=kDisableCompressionOption - prefix_extractor=nullptr - memtable_insert_with_hint_prefix_extractor=nullptr - memtable_factory=SkipListFactory - compaction_filter=nullptr - compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} - merge_operator=nullptr - compaction_style=kCompactionStyleLevel - -[TableOptions/BlockBasedTable "default"] - format_version=2 - whole_key_filtering=true - verify_compression=false - partition_filters=false - index_block_restart_interval=1 - block_size_deviation=10 - block_size=4096 - pin_l0_filter_and_index_blocks_in_cache=false - block_restart_interval=16 - filter_policy=nullptr - metadata_block_size=4096 - no_block_cache=false - checksum=kCRC32c - read_amp_bytes_per_bit=8589934592 - cache_index_and_filter_blocks=false - enable_index_compression=true - index_type=kBinarySearch - hash_index_allow_collision=true - cache_index_and_filter_blocks_with_high_priority=false - flush_block_policy_factory=FlushBlockBySizePolicyFactory - diff --git a/testdata/event_listener b/testdata/event_listener index 29228407174..0402b771090 100644 --- a/testdata/event_listener +++ b/testdata/event_listener @@ -7,14 +7,10 @@ open-dir: wal lock: db/LOCK open-dir: db open-dir: db -open: db/CURRENT create: db/MANIFEST-000001 sync: db/MANIFEST-000001 -remove: db/temporary.000001.dbtmp -create: db/temporary.000001.dbtmp -sync: db/temporary.000001.dbtmp -close: db/temporary.000001.dbtmp -rename: db/temporary.000001.dbtmp -> db/CURRENT +create: db/marker.manifest.000001.MANIFEST-000001 +close: db/marker.manifest.000001.MANIFEST-000001 sync: db [JOB 1] MANIFEST created 000001 open-dir: db @@ -22,86 +18,18 @@ sync: db/MANIFEST-000001 create: wal/000002.log sync: wal [JOB 1] WAL created 000002 -create: db/marker.manifest.000001.MANIFEST-000001 -close: db/marker.manifest.000001.MANIFEST-000001 -sync: db -create: db/marker.format-version.000001.002 -close: db/marker.format-version.000001.002 -sync: db -upgraded to format version: 002 -remove: db/temporary.000000.dbtmp -create: db/temporary.000000.dbtmp -sync: db/temporary.000000.dbtmp -close: db/temporary.000000.dbtmp -rename: db/temporary.000000.dbtmp -> db/CURRENT -create: db/marker.format-version.000002.003 -close: db/marker.format-version.000002.003 -remove: db/marker.format-version.000001.002 -sync: db -upgraded to format version: 003 -create: db/marker.format-version.000003.004 -close: db/marker.format-version.000003.004 -remove: db/marker.format-version.000002.003 -sync: db -upgraded to format version: 004 -create: db/marker.format-version.000004.005 -close: db/marker.format-version.000004.005 -remove: db/marker.format-version.000003.004 -sync: db -upgraded to format version: 005 -create: db/marker.format-version.000005.006 -close: db/marker.format-version.000005.006 -remove: db/marker.format-version.000004.005 -sync: db -upgraded to format version: 006 -create: db/marker.format-version.000006.007 -close: db/marker.format-version.000006.007 -remove: db/marker.format-version.000005.006 -sync: db -upgraded to format version: 007 -create: db/marker.format-version.000007.008 -close: db/marker.format-version.000007.008 -remove: db/marker.format-version.000006.007 -sync: db -upgraded to format version: 008 -create: db/marker.format-version.000008.009 -close: db/marker.format-version.000008.009 -remove: db/marker.format-version.000007.008 -sync: db -upgraded to format version: 009 -create: db/marker.format-version.000009.010 -close: db/marker.format-version.000009.010 -remove: db/marker.format-version.000008.009 -sync: db -upgraded to format version: 010 -create: db/marker.format-version.000010.011 -close: db/marker.format-version.000010.011 -remove: db/marker.format-version.000009.010 -sync: db -upgraded to format version: 011 -create: db/marker.format-version.000011.012 -close: db/marker.format-version.000011.012 -remove: db/marker.format-version.000010.011 -sync: db -upgraded to format version: 012 -create: db/marker.format-version.000012.013 -close: db/marker.format-version.000012.013 -remove: db/marker.format-version.000011.012 -sync: db -upgraded to format version: 013 -create: db/marker.format-version.000013.014 -close: db/marker.format-version.000013.014 -remove: db/marker.format-version.000012.013 +create: db/marker.format-version.000001.014 +close: db/marker.format-version.000001.014 sync: db upgraded to format version: 014 -create: db/marker.format-version.000014.015 -close: db/marker.format-version.000014.015 -remove: db/marker.format-version.000013.014 +create: db/marker.format-version.000002.015 +close: db/marker.format-version.000002.015 +remove: db/marker.format-version.000001.014 sync: db upgraded to format version: 015 -create: db/marker.format-version.000015.016 -close: db/marker.format-version.000015.016 -remove: db/marker.format-version.000014.015 +create: db/marker.format-version.000003.016 +close: db/marker.format-version.000003.016 +remove: db/marker.format-version.000002.015 sync: db upgraded to format version: 016 create: db/temporary.000003.dbtmp @@ -117,10 +45,10 @@ sync-data: wal/000002.log close: wal/000002.log create: wal/000004.log sync: wal -[JOB 4] WAL created 000004 -[JOB 5] flushing 1 memtable (100B) to L0 +[JOB 2] WAL created 000004 +[JOB 3] flushing 1 memtable (100B) to L0 create: db/000005.sst -[JOB 5] flushing: sstable created 000005 +[JOB 3] flushing: sstable created 000005 sync-data: db/000005.sst close: db/000005.sst sync: db @@ -131,8 +59,8 @@ create: db/marker.manifest.000002.MANIFEST-000006 close: db/marker.manifest.000002.MANIFEST-000006 remove: db/marker.manifest.000001.MANIFEST-000001 sync: db -[JOB 5] MANIFEST created 000006 -[JOB 5] flushed 1 memtable (100B) to L0 [000005] (662B), in 1.0s (2.0s total), output rate 662B/s +[JOB 3] MANIFEST created 000006 +[JOB 3] flushed 1 memtable (100B) to L0 [000005] (662B), in 1.0s (2.0s total), output rate 662B/s compact ---- @@ -141,10 +69,10 @@ sync-data: wal/000004.log close: wal/000004.log reuseForWrite: wal/000002.log -> wal/000007.log sync: wal -[JOB 6] WAL created 000007 (recycled 000002) -[JOB 7] flushing 1 memtable (100B) to L0 +[JOB 4] WAL created 000007 (recycled 000002) +[JOB 5] flushing 1 memtable (100B) to L0 create: db/000008.sst -[JOB 7] flushing: sstable created 000008 +[JOB 5] flushing: sstable created 000008 sync-data: db/000008.sst close: db/000008.sst sync: db @@ -155,11 +83,11 @@ create: db/marker.manifest.000003.MANIFEST-000009 close: db/marker.manifest.000003.MANIFEST-000009 remove: db/marker.manifest.000002.MANIFEST-000006 sync: db -[JOB 7] MANIFEST created 000009 -[JOB 7] flushed 1 memtable (100B) to L0 [000008] (662B), in 1.0s (2.0s total), output rate 662B/s +[JOB 5] MANIFEST created 000009 +[JOB 5] flushed 1 memtable (100B) to L0 [000008] (662B), in 1.0s (2.0s total), output rate 662B/s remove: db/MANIFEST-000001 -[JOB 7] MANIFEST deleted 000001 -[JOB 8] compacting(default) L0 [000005 000008] (1.3KB) Score=0.00 + L6 [] (0B) Score=0.00; OverlappingRatio: Single 0.00, Multi 0.00 +[JOB 5] MANIFEST deleted 000001 +[JOB 6] compacting(default) L0 [000005 000008] (1.3KB) Score=0.00 + L6 [] (0B) Score=0.00; OverlappingRatio: Single 0.00, Multi 0.00 open: db/000005.sst read-at(609, 53): db/000005.sst read-at(572, 37): db/000005.sst @@ -181,7 +109,7 @@ read-at(0, 26): db/000008.sst close: db/000008.sst close: db/000005.sst create: db/000010.sst -[JOB 8] compacting: sstable created 000010 +[JOB 6] compacting: sstable created 000010 sync-data: db/000010.sst close: db/000010.sst sync: db @@ -192,16 +120,16 @@ create: db/marker.manifest.000004.MANIFEST-000011 close: db/marker.manifest.000004.MANIFEST-000011 remove: db/marker.manifest.000003.MANIFEST-000009 sync: db -[JOB 8] MANIFEST created 000011 -[JOB 8] compacted(default) L0 [000005 000008] (1.3KB) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000010] (662B), in 1.0s (3.0s total), output rate 662B/s +[JOB 6] MANIFEST created 000011 +[JOB 6] compacted(default) L0 [000005 000008] (1.3KB) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000010] (662B), in 1.0s (3.0s total), output rate 662B/s close: db/000005.sst close: db/000008.sst remove: db/000005.sst -[JOB 8] sstable deleted 000005 +[JOB 6] sstable deleted 000005 remove: db/000008.sst -[JOB 8] sstable deleted 000008 +[JOB 6] sstable deleted 000008 remove: db/MANIFEST-000006 -[JOB 8] MANIFEST deleted 000006 +[JOB 6] MANIFEST deleted 000006 disable-file-deletions ---- @@ -213,10 +141,10 @@ sync-data: wal/000007.log close: wal/000007.log reuseForWrite: wal/000004.log -> wal/000012.log sync: wal -[JOB 9] WAL created 000012 (recycled 000004) -[JOB 10] flushing 1 memtable (100B) to L0 +[JOB 7] WAL created 000012 (recycled 000004) +[JOB 8] flushing 1 memtable (100B) to L0 create: db/000013.sst -[JOB 10] flushing: sstable created 000013 +[JOB 8] flushing: sstable created 000013 sync-data: db/000013.sst close: db/000013.sst sync: db @@ -227,13 +155,13 @@ create: db/marker.manifest.000005.MANIFEST-000014 close: db/marker.manifest.000005.MANIFEST-000014 remove: db/marker.manifest.000004.MANIFEST-000011 sync: db -[JOB 10] MANIFEST created 000014 -[JOB 10] flushed 1 memtable (100B) to L0 [000013] (662B), in 1.0s (2.0s total), output rate 662B/s +[JOB 8] MANIFEST created 000014 +[JOB 8] flushed 1 memtable (100B) to L0 [000013] (662B), in 1.0s (2.0s total), output rate 662B/s enable-file-deletions ---- remove: db/MANIFEST-000009 -[JOB 11] MANIFEST deleted 000009 +[JOB 9] MANIFEST deleted 000009 ingest ---- @@ -245,7 +173,7 @@ read-at(26, 27): ext/0 read-at(0, 26): ext/0 close: ext/0 link: ext/0 -> db/000015.sst -[JOB 12] ingesting: sstable created 000015 +[JOB 10] ingesting: sstable created 000015 sync: db open: db/000013.sst read-at(609, 53): db/000013.sst @@ -260,11 +188,11 @@ create: db/marker.manifest.000006.MANIFEST-000016 close: db/marker.manifest.000006.MANIFEST-000016 remove: db/marker.manifest.000005.MANIFEST-000014 sync: db -[JOB 12] MANIFEST created 000016 +[JOB 10] MANIFEST created 000016 remove: db/MANIFEST-000011 -[JOB 12] MANIFEST deleted 000011 +[JOB 10] MANIFEST deleted 000011 remove: ext/0 -[JOB 12] ingested L0:000015 (717B) +[JOB 10] ingested L0:000015 (717B) metrics ---- @@ -318,38 +246,38 @@ read-at(26, 27): ext/b read-at(0, 26): ext/b close: ext/b link: ext/a -> db/000017.sst -[JOB 13] ingesting: sstable created 000017 +[JOB 11] ingesting: sstable created 000017 link: ext/b -> db/000018.sst -[JOB 13] ingesting: sstable created 000018 +[JOB 11] ingesting: sstable created 000018 sync: db sync-data: wal/000012.log close: wal/000012.log reuseForWrite: wal/000007.log -> wal/000019.log sync: wal -[JOB 14] WAL created 000019 (recycled 000007) +[JOB 12] WAL created 000019 (recycled 000007) sync-data: wal/000019.log sync-data: wal/000019.log close: wal/000019.log create: wal/000020.log sync: wal -[JOB 15] WAL created 000020 +[JOB 13] WAL created 000020 remove: ext/a remove: ext/b -[JOB 13] ingested as flushable 000017 (717B), 000018 (717B) +[JOB 11] ingested as flushable 000017 (717B), 000018 (717B) sync-data: wal/000020.log close: wal/000020.log create: wal/000021.log sync: wal -[JOB 16] WAL created 000021 -[JOB 17] flushing 1 memtable (100B) to L0 +[JOB 14] WAL created 000021 +[JOB 15] flushing 1 memtable (100B) to L0 create: db/000022.sst -[JOB 17] flushing: sstable created 000022 +[JOB 15] flushing: sstable created 000022 sync-data: db/000022.sst close: db/000022.sst sync: db sync: db/MANIFEST-000016 -[JOB 17] flushed 1 memtable (100B) to L0 [000022] (662B), in 1.0s (2.0s total), output rate 662B/s -[JOB 18] flushing 2 ingested tables +[JOB 15] flushed 1 memtable (100B) to L0 [000022] (662B), in 1.0s (2.0s total), output rate 662B/s +[JOB 16] flushing 2 ingested tables create: db/MANIFEST-000023 close: db/MANIFEST-000016 sync: db/MANIFEST-000023 @@ -357,13 +285,13 @@ create: db/marker.manifest.000007.MANIFEST-000023 close: db/marker.manifest.000007.MANIFEST-000023 remove: db/marker.manifest.000006.MANIFEST-000016 sync: db -[JOB 18] MANIFEST created 000023 -[JOB 18] flushed 2 ingested flushables L0:000017 (717B) + L6:000018 (717B) in 1.0s (2.0s total), output rate 1.4KB/s +[JOB 16] MANIFEST created 000023 +[JOB 16] flushed 2 ingested flushables L0:000017 (717B) + L6:000018 (717B) in 1.0s (2.0s total), output rate 1.4KB/s remove: db/MANIFEST-000014 -[JOB 18] MANIFEST deleted 000014 -[JOB 19] flushing 1 memtable (100B) to L0 +[JOB 16] MANIFEST deleted 000014 +[JOB 17] flushing 1 memtable (100B) to L0 sync: db/MANIFEST-000023 -[JOB 19] flush error: pebble: empty table +[JOB 17] flush error: pebble: empty table metrics ---- diff --git a/testdata/external_iterator b/testdata/external_iterator index 589b950f244..e0e37bc080a 100644 --- a/testdata/external_iterator +++ b/testdata/external_iterator @@ -254,7 +254,7 @@ aaaa@1: (aaaa@1, .) aaaaa@3: (aaaaa@3, .) aaaaa@1: (aaaaa@1, .) stats: (interface (dir, seek, step): (fwd, 5, 5), (rev, 0, 0)), (internal (dir, seek, step): (fwd, 5, 5), (rev, 0, 0)), -(internal-stats: (block-bytes: (total 475B, cached 0B, read-time 0s)), (points: (count 10, key-bytes 50B, value-bytes 50B, tombstoned 0))) +(internal-stats: (block-bytes: (total 547B, cached 0B, read-time 0s)), (points: (count 10, key-bytes 50B, value-bytes 35B, tombstoned 0)), (separated: (count 15, bytes 65B, fetched 25B))) # Note the inclusion of fwd-only. This iterator will use the TrySeekUsingNext # optimization and loads ~half the block-bytes as a result. @@ -283,4 +283,4 @@ aaaa@1: (aaaa@1, .) aaaaa@3: (aaaaa@3, .) aaaaa@1: (aaaaa@1, .) stats: (interface (dir, seek, step): (fwd, 5, 5), (rev, 0, 0)), (internal (dir, seek, step): (fwd, 5, 5), (rev, 0, 0)), -(internal-stats: (block-bytes: (total 281B, cached 0B, read-time 0s)), (points: (count 10, key-bytes 50B, value-bytes 50B, tombstoned 0))) +(internal-stats: (block-bytes: (total 336B, cached 0B, read-time 0s)), (points: (count 10, key-bytes 50B, value-bytes 35B, tombstoned 0)), (separated: (count 5, bytes 25B, fetched 25B))) diff --git a/testdata/flushable_ingest b/testdata/flushable_ingest index ea20491fcd7..55c5b57ca44 100644 --- a/testdata/flushable_ingest +++ b/testdata/flushable_ingest @@ -56,12 +56,11 @@ ls 000006.sst 000007.log 000008.log -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000001.MANIFEST-000001 # Test basic WAL replay @@ -78,12 +77,11 @@ ls 000006.sst 000007.log 000008.log -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000001.MANIFEST-000001 open @@ -387,12 +385,11 @@ ls 000006.sst 000007.log 000008.log -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000001.MANIFEST-000001 close @@ -408,12 +405,11 @@ ls 000006.sst 000007.log 000008.log -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000001.MANIFEST-000001 open @@ -440,13 +436,12 @@ ls 000009.sst 000010.sst 000011.log -CURRENT LOCK MANIFEST-000001 MANIFEST-000012 OPTIONS-000013 ext -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000002.MANIFEST-000012 # Make sure that the new mutable memtable can accept writes. @@ -585,12 +580,11 @@ ls 000005.log 000006.log 000007.sst -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 ext -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000001.MANIFEST-000001 close @@ -604,13 +598,12 @@ resetToSynced 000004.sst 000005.log 000006.log -CURRENT LOCK MANIFEST-000001 OPTIONS-000003 ext ext1 -marker.format-version.000015.016 +marker.format-version.000003.016 marker.manifest.000001.MANIFEST-000001 ignoreSyncs false diff --git a/testdata/format_major_version_pebblev1_migration b/testdata/format_major_version_pebblev1_migration deleted file mode 100644 index c579a530500..00000000000 --- a/testdata/format_major_version_pebblev1_migration +++ /dev/null @@ -1,170 +0,0 @@ -# Open the DB at one version prior to the version that enforces Pebblev1 tables. -open version=8 ----- - -format-major-version ----- -008 - -# Confirm the allowable range of table formats. - -min-table-format ----- -(LevelDB) - -max-table-format ----- -(Pebble,v2) - -# Disable automatic compactions while we create the tables. - -disable-automatic-compactions true ----- - -# Create and flush a table. The table is created at the max format version for -# this DB version (i.e. Pebblev2). - -batch -set a a ----- - -flush ----- - -# Ingest an external table written at the max table format for the current -# version (i.e. Pebblev2). - -ingest a format=pebblev2 -set pebblev2 pebblev2 ----- - -# Ingest some external table written at earlier versions (i.e. Pebblev1, -# RocksDBv2, LevelDB). - -ingest b format=pebblev1 -set pebblev1 pebblev1 ----- - -ingest c format=rocksdbv2 -set rocksdbv2 rockdbv2 ----- - -ingest d format=leveldb -set leveldb leveldb ----- - -lsm ----- -0.0: - 000005:[a#10,SET-a#10,SET] -6: - 000009:[leveldb#14,SET-leveldb#14,SET] - 000007:[pebblev1#12,SET-pebblev1#12,SET] - 000006:[pebblev2#11,SET-pebblev2#11,SET] - 000008:[rocksdbv2#13,SET-rocksdbv2#13,SET] - -tally-table-formats ----- -(LevelDB): 1 -(RocksDB,v2): 1 -(Pebble,v1): 1 -(Pebble,v2): 2 -(Pebble,v3): 0 -(Pebble,v4): 0 - -# Upgrade the DB to FormatMinTableFormatPebblev1. - -ratchet-format-major-version 009 ----- - -format-major-version ----- -009 - -# The min table format version has been raised to Pebblev1. - -min-table-format ----- -(Pebble,v1) - -max-table-format ----- -(Pebble,v2) - -# Ingesting a table with a format prior to this version fails. - -ingest e format=rocksdbv2 -set rocksdbv2 rockdbv2 ----- -pebble: table format (RocksDB,v2) is not within range supported at DB format major version 9, ((Pebble,v1),(Pebble,v2)) - -# Upgrade the DB to FormatPrePebblev1Marked. The marked count increases to the -# count of tables at versions pre-Pebblev1 (i.e. two tables). - -ratchet-format-major-version 010 ----- - -format-major-version ----- -010 - -min-table-format ----- -(Pebble,v1) - -max-table-format ----- -(Pebble,v2) - -marked-file-count ----- -2 files marked for compaction - -# Upgrade the DB to FormatPrePebblev1MarkedCompacted. The marked count returns -# to zero. - -disable-automatic-compactions false ----- - -ratchet-format-major-version 014 ----- - -format-major-version ----- -014 - -min-table-format ----- -(Pebble,v1) - -max-table-format ----- -(Pebble,v3) - -marked-file-count ----- -0 files marked for compaction - -# The two tables with older table formats were rewritten with newer table format -# versions (note updated table numbers for the leveldb and rocksdb2 tables). - -lsm ----- -0.0: - 000005:[a#10,SET-a#10,SET] -6: - 000013:[leveldb#0,SET-leveldb#0,SET] - 000007:[pebblev1#12,SET-pebblev1#12,SET] - 000006:[pebblev2#11,SET-pebblev2#11,SET] - 000012:[rocksdbv2#0,SET-rocksdbv2#0,SET] - -# Confirm all tables are at least the minimum supported table format version. - -tally-table-formats ----- -(LevelDB): 0 -(RocksDB,v2): 0 -(Pebble,v1): 1 -(Pebble,v2): 4 -(Pebble,v3): 0 -(Pebble,v4): 0 diff --git a/testdata/format_major_version_split_user_key_migration b/testdata/format_major_version_split_user_key_migration deleted file mode 100644 index 735e7c02080..00000000000 --- a/testdata/format_major_version_split_user_key_migration +++ /dev/null @@ -1,148 +0,0 @@ -define -L1 -d.SET.110:d e.SET.140:e ----- -1: - 000004:[d#110,SET-e#140,SET] seqnums:[110-140] points:[d#110,SET-e#140,SET] - -reopen ----- -OK - -# The current public Pebble interface offers no way of constructing a multi-file -# atomic compaction unit, so use the force-ingest command to force an ingestion -# into L1. - -build cd -set c c -set d d ----- - -force-ingest paths=(cd) level=1 ----- -1: - 000008:[c#141,SET-d#141,SET] seqnums:[141-141] points:[c#141,SET-d#141,SET] - 000004:[d#110,SET-e#140,SET] seqnums:[110-140] points:[d#110,SET-e#140,SET] - -format-major-version ----- -005 - -marked-file-count ----- -0 files marked for compaction - -ratchet-format-major-version 006 ----- - -format-major-version ----- -006 - -# Upgrading to format major version 006 should've marked files for compaction. - -marked-file-count ----- -2 files marked for compaction - -reopen ----- -OK - -format-major-version ----- -006 - -# Ensure the files previously marked for compaction are still marked for -# compaction. - -marked-file-count ----- -2 files marked for compaction - -disable-automatic-compactions false ----- - -# Ratcheting to 007 should force compaction of any files still marked for -# compaction. - -ratchet-format-major-version 007 ----- -[JOB 100] compacted(rewrite) L1 [000008 000004] (1.3KB) Score=0.00 + L1 [] (0B) Score=0.00 -> L1 [000013] (649B), in 1.0s (2.0s total), output rate 649B/s - -format-major-version ----- -007 - -lsm ----- -1: - 000013:[c#0,SET-e#0,SET] - -# Reset to a new LSM. - -define -L1 -b.SET.0:b c.SET.5:c -L1 -m.SET.0:m l.SET.5:l -L1 -x.SET.0:x y.SET.5:y ----- -1: - 000004:[b#0,SET-c#5,SET] seqnums:[0-5] points:[b#0,SET-c#5,SET] - 000005:[l#5,SET-m#0,SET] seqnums:[0-5] points:[l#5,SET-m#0,SET] - 000006:[x#0,SET-y#5,SET] seqnums:[0-5] points:[x#0,SET-y#5,SET] - -build ab -set a a -set b b ----- - -build wx -set w w -set x x ----- - -force-ingest paths=(ab, wx) level=1 ----- -1: - 000007:[a#10,SET-b#10,SET] seqnums:[10-10] points:[a#10,SET-b#10,SET] - 000004:[b#0,SET-c#5,SET] seqnums:[0-5] points:[b#0,SET-c#5,SET] - 000005:[l#5,SET-m#0,SET] seqnums:[0-5] points:[l#5,SET-m#0,SET] - 000008:[w#11,SET-x#11,SET] seqnums:[11-11] points:[w#11,SET-x#11,SET] - 000006:[x#0,SET-y#5,SET] seqnums:[0-5] points:[x#0,SET-y#5,SET] - -format-major-version ----- -005 - -ratchet-format-major-version 006 ----- - -format-major-version ----- -006 - -marked-file-count ----- -4 files marked for compaction - -disable-automatic-compactions false ----- - -ratchet-format-major-version 007 ----- -[JOB 100] compacted(rewrite) L1 [000007 000004] (1.3KB) Score=0.00 + L1 [] (0B) Score=0.00 -> L1 [000010] (649B), in 1.0s (2.0s total), output rate 649B/s -[JOB 100] compacted(rewrite) L1 [000008 000006] (1.3KB) Score=0.00 + L1 [] (0B) Score=0.00 -> L1 [000011] (649B), in 1.0s (2.0s total), output rate 649B/s - -lsm ----- -1: - 000010:[a#0,SET-c#0,SET] - 000005:[l#5,SET-m#0,SET] - 000011:[w#0,SET-y#0,SET] - -format-major-version ----- -007 diff --git a/testdata/ingest_load b/testdata/ingest_load index 3e5d690c729..72b2a064b0f 100644 --- a/testdata/ingest_load +++ b/testdata/ingest_load @@ -89,15 +89,15 @@ a.RANGEDEL.0:b ranges: #0,0-#0,0 # Loading tables at an unsupported table format results in an error. -# Write a table at version 7 (Pebble,v2) into a DB at version 6 (Pebble,v1). -load writer-version=8 db-version=7 +# Write a table at version 15 (Pebble,v4) into a DB at version 14 (Pebble,v3). +load writer-version=15 db-version=14 a.SET.1: ---- -pebble: table format (Pebble,v2) is not within range supported at DB format major version 7, ((LevelDB),(Pebble,v1)) +pebble: table format (Pebble,v4) is not within range supported at DB format major version 14, ((Pebble,v1),(Pebble,v3)) # Tables with range keys only. -load writer-version=10 db-version=10 +load writer-version=16 db-version=16 rangekey: a-z:{(#0,RANGEKEYSET,@1,foo)} ---- 1: a#0,21-z#72057594037927935,21 @@ -106,7 +106,7 @@ rangekey: a-z:{(#0,RANGEKEYSET,@1,foo)} # Tables with a mixture of point and range keys. -load writer-version=10 db-version=10 +load writer-version=16 db-version=16 a.SET.0: b.SET.0: c.SET.0: @@ -118,7 +118,7 @@ rangekey: y-z:{(#0,RANGEKEYSET,@3,baz)} points: a#0,1-c#0,1 ranges: w#0,21-z#72057594037927935,21 -load writer-version=10 db-version=10 +load writer-version=16 db-version=16 c.SET.0:d rangekey: a-z:{(#0,RANGEKEYSET,@1,foo)} ---- @@ -126,7 +126,7 @@ rangekey: a-z:{(#0,RANGEKEYSET,@1,foo)} points: c#0,1-c#0,1 ranges: a#0,21-z#72057594037927935,21 -load writer-version=10 db-version=10 +load writer-version=16 db-version=16 a.SET.0:z rangekey: c-d:{(#0,RANGEKEYSET,@1,foo)} ---- @@ -136,7 +136,7 @@ rangekey: c-d:{(#0,RANGEKEYSET,@1,foo)} # NB: range dels sort before range keys -load writer-version=10 db-version=10 +load writer-version=16 db-version=16 a.RANGEDEL.0:z rangekey: a-z:{(#0,RANGEKEYSET,@1,foo)} ---- diff --git a/testdata/iter_histories/range_keys_simple b/testdata/iter_histories/range_keys_simple index 6748821e702..e65712d5e7c 100644 --- a/testdata/iter_histories/range_keys_simple +++ b/testdata/iter_histories/range_keys_simple @@ -414,26 +414,6 @@ z@10: (z@10, .) z@1: (z@1, .) za@100: (za@100, .) -# Applying range keys to a DB running with a version that doesn't support them -# results in an error. Range keys were added in version 7. -reset format-major-version=6 ----- - -batch commit -range-key-set a z @5 boop ----- -pebble: batch requires at least format major version 8 (current: 6) - -# Constructing iterator over range keys on a DB that doesn't support them -# results in an error. - -reset format-major-version=6 ----- - -combined-iter ----- -pebble: range keys require at least format major version 8 (current: 6) - # Test Prev-ing back over a synthetic range key marker. Synthetic range-key # markers (the keys interleaved at 'c' during a SeekGE(c) when there's a # straddling range key) are ephemeral, and Prev-ing back must move back the diff --git a/testdata/manual_compaction b/testdata/manual_compaction deleted file mode 100644 index cd3ad5f9578..00000000000 --- a/testdata/manual_compaction +++ /dev/null @@ -1,1236 +0,0 @@ -batch -set a 1 -set b 2 ----- - -compact a-b ----- -6: - 000005:[a#10,SET-b#11,SET] - -batch -set c 3 -set d 4 ----- - -compact c-d ----- -6: - 000005:[a#10,SET-b#11,SET] - 000007:[c#12,SET-d#13,SET] - -batch -set b 5 -set c 6 ----- - -compact a-d ----- -6: - 000010:[a#0,SET-d#0,SET] - -# This also tests flushing a memtable that only contains range -# deletions. - -batch -del-range a e ----- - -compact a-d ----- - -# Test that a multi-output-file compaction generates non-overlapping files. - -define target-file-sizes=(100, 1) -L0 - b.SET.1:v -L0 - a.SET.2:v ----- -0.0: - 000005:[a#2,SET-a#2,SET] - 000004:[b#1,SET-b#1,SET] - -compact a-b ----- -1: - 000006:[a#0,SET-a#0,SET] - 000007:[b#0,SET-b#0,SET] - -# A range tombstone extends past the grandparent file boundary used to limit the -# size of future compactions. Verify the range tombstone is split at that file -# boundary. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.SET.3:v -L2 - a.RANGEDEL.2:e -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v - d.SET.0:v ----- -1: - 000004:[a#3,SET-a#3,SET] -2: - 000005:[a#2,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - -wait-pending-table-stats -000005 ----- -num-entries: 1 -num-deletions: 1 -num-range-key-sets: 0 -point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1552 - -compact a-e L1 ----- -2: - 000008:[a#3,SET-c#inf,RANGEDEL] - 000009:[c#2,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - -wait-pending-table-stats -000008 ----- -num-entries: 2 -num-deletions: 1 -num-range-key-sets: 0 -point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 776 - -# Same as above, except range tombstone covers multiple grandparent file boundaries. - -define target-file-sizes=(1, 1, 1, 1) format-major-version=1 -L1 - a.SET.3:v -L2 - a.RANGEDEL.2:g -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v - d.SET.0:v -L3 - e.SET.0:v - f.SET.1:v -L3 - f.SET.0:v - g.SET.0:v ----- -1: - 000004:[a#3,SET-a#3,SET] -2: - 000005:[a#2,RANGEDEL-g#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - 000008:[e#0,SET-f#1,SET] - 000009:[f#0,SET-g#0,SET] - -compact a-e L1 ----- -2: - 000010:[a#3,SET-c#inf,RANGEDEL] - 000011:[c#2,RANGEDEL-e#inf,RANGEDEL] - 000012:[e#2,RANGEDEL-f#inf,RANGEDEL] - 000013:[f#2,RANGEDEL-g#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - 000008:[e#0,SET-f#1,SET] - 000009:[f#0,SET-g#0,SET] - -# A range tombstone covers multiple grandparent file boundaries between point keys, -# rather than after all point keys. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.SET.3:v - h.SET.3:v -L2 - a.RANGEDEL.2:g -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v - d.SET.0:v -L3 - e.SET.0:v - f.SET.1:v ----- -1: - 000004:[a#3,SET-h#3,SET] -2: - 000005:[a#2,RANGEDEL-g#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - 000008:[e#0,SET-f#1,SET] - -compact a-e L1 ----- -2: - 000009:[a#3,SET-c#inf,RANGEDEL] - 000010:[c#2,RANGEDEL-h#3,SET] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - 000008:[e#0,SET-f#1,SET] - -# A range tombstone is the first and only item output by a compaction, and it -# extends past the grandparent file boundary used to limit the size of future -# compactions. Verify the range tombstone is split at that file boundary. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.RANGEDEL.3:e -L2 - a.SET.2:v -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v - d.SET.0:v ----- -1: - 000004:[a#3,RANGEDEL-e#inf,RANGEDEL] -2: - 000005:[a#2,SET-a#2,SET] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - -compact a-e L1 ----- -2: - 000008:[a#3,RANGEDEL-c#inf,RANGEDEL] - 000009:[c#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - -# An elided range tombstone is the first item encountered by a compaction, -# and the grandparent limit set by it extends to the next item, also a range -# tombstone. The first item should be elided, and the second item should -# reset the grandparent limit. - -define target-file-sizes=(100, 100, 100, 100) -L1 - a.RANGEDEL.4:d -L1 - grandparent.RANGEDEL.2:z - h.SET.3:v -L2 - grandparent.SET.1:v -L3 - grandparent.SET.0:v -L3 - m.SET.0:v ----- -1: - 000004:[a#4,RANGEDEL-d#inf,RANGEDEL] - 000005:[grandparent#2,RANGEDEL-z#inf,RANGEDEL] -2: - 000006:[grandparent#1,SET-grandparent#1,SET] -3: - 000007:[grandparent#0,SET-grandparent#0,SET] - 000008:[m#0,SET-m#0,SET] - -compact a-h L1 ----- -2: - 000009:[grandparent#2,RANGEDEL-m#inf,RANGEDEL] - 000010:[m#2,RANGEDEL-z#inf,RANGEDEL] -3: - 000007:[grandparent#0,SET-grandparent#0,SET] - 000008:[m#0,SET-m#0,SET] - -# Setup such that grandparent overlap limit is exceeded multiple times at the same user key ("b"). -# Ensures the compaction output files are non-overlapping. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.SET.2:v - c.SET.2:v -L2 - a.RANGEDEL.3:c -L3 - b.SET.2:v -L3 - b.SET.1:v -L3 - b.SET.0:v ----- -1: - 000004:[a#2,SET-c#2,SET] -2: - 000005:[a#3,RANGEDEL-c#inf,RANGEDEL] -3: - 000006:[b#2,SET-b#2,SET] - 000007:[b#1,SET-b#1,SET] - 000008:[b#0,SET-b#0,SET] - -compact a-c L1 ----- -2: - 000009:[a#3,RANGEDEL-b#inf,RANGEDEL] - 000010:[b#3,RANGEDEL-c#2,SET] -3: - 000006:[b#2,SET-b#2,SET] - 000007:[b#1,SET-b#1,SET] - 000008:[b#0,SET-b#0,SET] - -# Regression test for a bug where compaction would stop process range -# tombstones for an input level upon finding an sstable in the input -# level with no range tombstones. In the scenario below, sstable 6 -# does not contain any range tombstones while sstable 7 does. Both are -# compacted together with sstable 5. - -reset ----- - -batch -set a 1 -set b 1 -set c 1 -set d 1 -set z 1 ----- - -compact a-z ----- -6: - 000005:[a#10,SET-z#14,SET] - -build ext1 -set a 2 ----- - -build ext2 -set b 2 -del-range c z ----- - -ingest ext1 ext2 ----- -0.0: - 000006:[a#15,SET-a#15,SET] - 000007:[b#16,SET-z#inf,RANGEDEL] -6: - 000005:[a#10,SET-z#14,SET] - -iter -first -next -next -next ----- -a: (2, .) -b: (2, .) -z: (1, .) -. - -compact a-z ----- -6: - 000008:[a#0,SET-z#0,SET] - -iter -first -next -next -next ----- -a: (2, .) -b: (2, .) -z: (1, .) -. - -# Regresion test for a bug in sstable smallest boundary generation -# where the smallest key for an sstable was set to a key "larger" than -# the start key of the first range tombstone. This in turn fouled up -# the processing logic of range tombstones used by mergingIter which -# allowed stepping out of an sstable even though it contained a range -# tombstone that covered keys in lower levels. - -define target-file-sizes=(1, 1, 1, 1) -L0 - c.SET.4:4 -L1 - a.SET.3:3 -L2 - a.RANGEDEL.2:e -L3 - b.SET.1:1 ----- -0.0: - 000004:[c#4,SET-c#4,SET] -1: - 000005:[a#3,SET-a#3,SET] -2: - 000006:[a#2,RANGEDEL-e#inf,RANGEDEL] -3: - 000007:[b#1,SET-b#1,SET] - -compact a-e L1 ----- -0.0: - 000004:[c#4,SET-c#4,SET] -2: - 000008:[a#3,SET-b#inf,RANGEDEL] - 000009:[b#2,RANGEDEL-e#inf,RANGEDEL] -3: - 000007:[b#1,SET-b#1,SET] - -# We should only see a:3 and c:4 at this point. - -iter -first -next -next ----- -a: (3, .) -c: (4, .) -. - -# The bug allowed seeing b:1 during reverse iteration. - -iter -last -prev -prev ----- -c: (4, .) -a: (3, .) -. - -# This is a similar scenario to the one above. In older versions of Pebble this -# case necessitated adjusting the seqnum of the range tombstone to -# prev.LargestKey.SeqNum-1. We no longer allow user keys to be split across -# sstables, and the seqnum adjustment is no longer necessary. -# -# Note the target-file-size of 26 is specially tailored to get the -# desired compaction output. - -define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) -L1 - a.SET.4:4 -L1 - b.SET.2:2 - b.RANGEDEL.3:e -L3 - b.SET.1:1 ----- -1: - 000004:[a#4,SET-a#4,SET] - 000005:[b#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -compact a-e L1 ----- -2: - 000007:[a#4,SET-a#4,SET] - 000008:[b#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -iter -first -next -last -prev ----- -a: (4, .) -. -a: (4, .) -. - -# Similar to the preceding scenario, except the range tombstone has -# the same seqnum as the largest key in the preceding file. - -define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) -L1 - a.SET.4:4 -L1 - b.SET.3:3 - b.RANGEDEL.3:e -L3 - b.SET.1:1 ----- -1: - 000004:[a#4,SET-a#4,SET] - 000005:[b#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -compact a-e L1 ----- -2: - 000007:[a#4,SET-a#4,SET] - 000008:[b#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -iter -first -next -next -last -prev -prev ----- -a: (4, .) -b: (3, .) -. -b: (3, .) -a: (4, .) -. - -# Similar to the preceding scenario, except the range tombstone has -# a smaller seqnum than the largest key in the preceding file. - -define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) -L1 - a.SET.4:4 -L1 - b.SET.4:4 - b.RANGEDEL.2:e -L3 - b.SET.1:1 ----- -1: - 000004:[a#4,SET-a#4,SET] - 000005:[b#4,SET-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -compact a-e L1 ----- -2: - 000007:[a#4,SET-a#4,SET] - 000008:[b#4,SET-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -iter -first -next -next -last -prev -prev ----- -a: (4, .) -b: (4, .) -. -b: (4, .) -a: (4, .) -. - -# Test a scenario where the last point key in an sstable has a seqnum -# of 0. - -define target-file-sizes=(1, 1, 26) snapshots=(2) -L1 - a.SET.3:3 - b.RANGEDEL.3:e - b.SET.0:0 -L3 - a.RANGEDEL.2:b -L3 - c.SET.0:0 - d.SET.0:0 ----- -1: - 000004:[a#3,SET-e#inf,RANGEDEL] -3: - 000005:[a#2,RANGEDEL-b#inf,RANGEDEL] - 000006:[c#0,SET-d#0,SET] - -iter -last -prev ----- -a: (3, .) -. - -compact a-e L1 ----- -2: - 000007:[a#3,SET-c#inf,RANGEDEL] - 000008:[c#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000005:[a#2,RANGEDEL-b#inf,RANGEDEL] - 000006:[c#0,SET-d#0,SET] - -iter -last -prev ----- -a: (3, .) -. - -# Test a scenario where the last point key in an sstable before the -# grandparent limit is reached has a seqnum of 0. We want to cut the -# sstable after the next point key is added, rather than continuing to -# add keys indefinitely (or till the size limit is reached). - -define target-file-sizes=(100, 1, 52) snapshots=(2) -L1 - a.SET.3:3 - b.RANGEDEL.3:e - b.SET.0:0 - c.SET.3:1 - d.SET.1:1 -L3 - c.RANGEDEL.2:d ----- -1: - 000004:[a#3,SET-e#inf,RANGEDEL] -3: - 000005:[c#2,RANGEDEL-d#inf,RANGEDEL] - -compact a-f L1 ----- -2: - 000006:[a#3,SET-c#inf,RANGEDEL] - 000007:[c#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000005:[c#2,RANGEDEL-d#inf,RANGEDEL] - -# Test a scenario where we the last point key in an sstable has a -# seqnum of 0, but there is another range tombstone later in the -# compaction. This scenario was previously triggering an assertion due -# to the rangedel.Fragmenter being finished prematurely. - -define target-file-sizes=(1, 1, 1) -L1 - a.SET.0:0 - c.RANGEDEL.1:d -L3 - b.SET.0:0 ----- -1: - 000004:[a#0,SET-d#inf,RANGEDEL] -3: - 000005:[b#0,SET-b#0,SET] - -compact a-e L1 ----- -2: - 000006:[a#0,SET-a#0,SET] -3: - 000005:[b#0,SET-b#0,SET] - -define target-file-sizes=(1, 1, 1, 1) -L0 - b.SET.1:v -L0 - a.SET.2:v ----- -0.0: - 000005:[a#2,SET-a#2,SET] - 000004:[b#1,SET-b#1,SET] - -add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=b ----- - -async-compact a-b L0 ----- -manual compaction blocked until ongoing finished -1: - 000006:[a#0,SET-a#0,SET] - 000007:[b#0,SET-b#0,SET] - -compact a-b L1 ----- -2: - 000008:[a#0,SET-a#0,SET] - 000009:[b#0,SET-b#0,SET] - -add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=b ----- - -async-compact a-b L2 ----- -manual compaction blocked until ongoing finished -3: - 000010:[a#0,SET-a#0,SET] - 000011:[b#0,SET-b#0,SET] - -add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=b ----- - -set-concurrent-compactions num=2 ----- - -async-compact a-b L3 ----- -manual compaction did not block for ongoing -4: - 000012:[a#0,SET-a#0,SET] - 000013:[b#0,SET-b#0,SET] - -remove-ongoing-compaction ----- - -add-ongoing-compaction startLevel=4 outputLevel=5 start=a end=b ----- - -async-compact a-b L4 ----- -manual compaction blocked until ongoing finished -5: - 000014:[a#0,SET-a#0,SET] - 000015:[b#0,SET-b#0,SET] - -# Test of a scenario where consecutive elided range tombstones and grandparent -# boundaries could result in an invariant violation in the rangedel fragmenter. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.RANGEDEL.4:b - c.RANGEDEL.4:d - e.RANGEDEL.4:f -L1 - g.RANGEDEL.6:h - i.RANGEDEL.4:j -L1 - k.RANGEDEL.5:q - m.RANGEDEL.4:q -L2 - a.SET.2:foo -L3 - a.SET.1:foo - c.SET.1:foo -L3 - ff.SET.1:v -L3 - k.SET.1:foo ----- -1: - 000004:[a#4,RANGEDEL-f#inf,RANGEDEL] - 000005:[g#6,RANGEDEL-j#inf,RANGEDEL] - 000006:[k#5,RANGEDEL-q#inf,RANGEDEL] -2: - 000007:[a#2,SET-a#2,SET] -3: - 000008:[a#1,SET-c#1,SET] - 000009:[ff#1,SET-ff#1,SET] - 000010:[k#1,SET-k#1,SET] - -compact a-q L1 ----- -2: - 000011:[a#4,RANGEDEL-d#inf,RANGEDEL] - 000012:[k#5,RANGEDEL-m#inf,RANGEDEL] -3: - 000008:[a#1,SET-c#1,SET] - 000009:[ff#1,SET-ff#1,SET] - 000010:[k#1,SET-k#1,SET] - -# Test a case where a new output file is started, there are no previous output -# files, there are no additional keys (key = nil) and the rangedel fragmenter -# is non-empty. -define target-file-sizes=(1, 1, 1) -L1 - a.RANGEDEL.10:b - d.RANGEDEL.9:e - q.RANGEDEL.8:r -L2 - g.RANGEDEL.7:h -L3 - q.SET.6:6 ----- -1: - 000004:[a#10,RANGEDEL-r#inf,RANGEDEL] -2: - 000005:[g#7,RANGEDEL-h#inf,RANGEDEL] -3: - 000006:[q#6,SET-q#6,SET] - -compact a-r L1 ----- -2: - 000007:[q#8,RANGEDEL-r#inf,RANGEDEL] -3: - 000006:[q#6,SET-q#6,SET] - -define target-file-sizes=(100, 100, 100) -L1 - a.RANGEDEL.10:b - b.SET.0:foo - d.RANGEDEL.0:e - j.SET.10:foo -L2 - f.RANGEDEL.7:g -L3 - c.SET.6:6 -L3 - c.SET.5:5 -L3 - c.SET.4:4 -L4 - a.SET.0:0 - f.SET.0:0 ----- -1: - 000004:[a#10,RANGEDEL-j#10,SET] -2: - 000005:[f#7,RANGEDEL-g#inf,RANGEDEL] -3: - 000006:[c#6,SET-c#6,SET] - 000007:[c#5,SET-c#5,SET] - 000008:[c#4,SET-c#4,SET] -4: - 000009:[a#0,SET-f#0,SET] - -compact a-r L1 ----- -2: - 000010:[a#10,RANGEDEL-b#0,SET] - 000011:[d#0,RANGEDEL-j#10,SET] -3: - 000006:[c#6,SET-c#6,SET] - 000007:[c#5,SET-c#5,SET] - 000008:[c#4,SET-c#4,SET] -4: - 000009:[a#0,SET-f#0,SET] - -# Test a snapshot that separates a range deletion from all the data that it -# deletes. Ensure that we respect the target-file-size and split into multiple -# outputs. - -define target-file-sizes=(1, 1, 1) snapshots=(14) -L1 - a.RANGEDEL.15:z - b.SET.11:foo - c.SET.11:foo -L2 - c.SET.0:foo - d.SET.0:foo ----- -1: - 000004:[a#15,RANGEDEL-z#inf,RANGEDEL] -2: - 000005:[c#0,SET-d#0,SET] - -sstable-properties file=000004 -snapshot-pinned-keys ----- -snapshot-pinned-keys: - pebble.num.snapshot-pinned-keys: 2 - pebble.raw.snapshot-pinned-keys.size: 18 - -compact a-z L1 ----- -2: - 000006:[a#15,RANGEDEL-c#inf,RANGEDEL] - 000007:[c#15,RANGEDEL-d#inf,RANGEDEL] - 000008:[d#15,RANGEDEL-z#inf,RANGEDEL] - -sstable-properties file=000006 -snapshot-pinned-keys ----- -snapshot-pinned-keys: - pebble.num.snapshot-pinned-keys: 1 - pebble.raw.snapshot-pinned-keys.size: 9 - -sstable-properties file=000007 -snapshot-pinned-keys ----- -snapshot-pinned-keys: - pebble.num.snapshot-pinned-keys: 1 - pebble.raw.snapshot-pinned-keys.size: 9 - -sstable-properties file=000008 -snapshot-pinned-keys ----- -snapshot-pinned-keys: - pebble.num.snapshot-pinned-keys: 1 - pebble.raw.snapshot-pinned-keys.size: 9 - -# Test an interaction between a range deletion that will be elided with -# output splitting. Ensure that the output is still split (previous versions -# of the code did not, because of intricacies around preventing a zero -# sequence number in an output's largest key). - -define target-file-sizes=(1, 1, 1) -L1 - a.RANGEDEL.10:z - b.SET.11:foo - c.SET.11:foo -L2 - c.SET.0:foo - d.SET.0:foo ----- -1: - 000004:[a#10,RANGEDEL-z#inf,RANGEDEL] -2: - 000005:[c#0,SET-d#0,SET] - -compact a-z L1 ----- -2: - 000006:[b#0,SET-b#0,SET] - 000007:[c#0,SET-c#0,SET] - -define target-file-sizes=(1, 1, 1, 1) -L0 - a.SET.3:v - b.SET.2:v -L2 - a.SET.1:v -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v ----- -0.0: - 000004:[a#3,SET-b#2,SET] -2: - 000005:[a#1,SET-a#1,SET] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-c#0,SET] - -set-concurrent-compactions num=3 ----- - -compact a-c parallel hide-file-num ----- -4: - [a#0,SET-a#0,SET] - [b#0,SET-b#0,SET] - [c#0,SET-c#0,SET] - -define target-file-sizes=(1, 1, 1, 1) -L0 - a.SET.3:v - b.SET.2:v -L0 - a.SET.2:v - c.SET.2:v -L2 - a.SET.1:v - b.SET.1:v -L2 - c.SET.1:v -L2 - d.SET.0:v -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v ----- -0.1: - 000004:[a#3,SET-b#2,SET] -0.0: - 000005:[a#2,SET-c#2,SET] -2: - 000006:[a#1,SET-b#1,SET] - 000007:[c#1,SET-c#1,SET] - 000008:[d#0,SET-d#0,SET] -3: - 000009:[a#0,SET-b#0,SET] - 000010:[c#0,SET-c#0,SET] - -set-concurrent-compactions num=2 ----- - -compact a-c L0 parallel ----- -1: - 000011:[a#3,SET-a#3,SET] - 000012:[b#2,SET-b#2,SET] - 000013:[c#2,SET-c#2,SET] -2: - 000006:[a#1,SET-b#1,SET] - 000007:[c#1,SET-c#1,SET] - 000008:[d#0,SET-d#0,SET] -3: - 000009:[a#0,SET-b#0,SET] - 000010:[c#0,SET-c#0,SET] - -add-ongoing-compaction startLevel=3 outputLevel=4 start=a end=d ----- - -# We allow 2 maximum concurrent compactions. The operation below generates -# 2 concurrent compactions (a-b, c) from L1 to L2. With 1 ongoing compaction with -# output level L4, there is no conflict and thus the concurrent compactions should -# be queued up and executed sequentially. We ensure that the compactions finish and -# that the final result of the compactions is correct. - -async-compact a-d L1 parallel ----- -manual compaction did not block for ongoing -2: - 000014:[a#3,SET-a#3,SET] - 000015:[b#2,SET-b#2,SET] - 000016:[c#2,SET-c#2,SET] - 000008:[d#0,SET-d#0,SET] -3: - 000009:[a#0,SET-b#0,SET] - 000010:[c#0,SET-c#0,SET] - -remove-ongoing-compaction ----- - -set-concurrent-compactions num=3 ----- - -compact a-d parallel hide-file-num ----- -4: - [a#0,SET-a#0,SET] - [b#0,SET-b#0,SET] - [c#0,SET-c#0,SET] - [d#0,SET-d#0,SET] - -# Create a contrived compaction that forces point key and rangedel iterators -# to stay in sync to emit a correct view of visible and deleted keys. Note that -# Pebble no longer produces range tombstones that go outside a file's bounds, -# but past versions of pebble did, and we should still be able to handle those -# well. - -define target-file-sizes=(1, 1, 1, 1, 1, 1) snapshots=(390) -L3 start=tmgc.MERGE.391 end=tmgc.MERGE.391 - tmgc.MERGE.391:foo - tmgc.RANGEDEL.331:udkatvs -L3 start=tmgc.MERGE.384 end=tmgc.MERGE.384 - tmgc.MERGE.384:bar - tmgc.RANGEDEL.383:tvsalezade - tmgc.RANGEDEL.331:tvsalezade -L3 start=tmgc.RANGEDEL.383 end=tvsalezade.RANGEDEL.72057594037927935 - tmgc.RANGEDEL.383:tvsalezade - tmgc.SET.375:baz - tmgc.RANGEDEL.356:tvsalezade ----- -3: - 000004:[tmgc#391,MERGE-tmgc#391,MERGE] - 000005:[tmgc#384,MERGE-tmgc#384,MERGE] - 000006:[tmgc#383,RANGEDEL-tvsalezade#inf,RANGEDEL] - -compact a-z L3 ----- -4: - 000007:[tmgc#391,MERGE-tmgc#384,MERGE] - -# baz should NOT be visible in the value. - -iter -first -next -next ----- -tmgc: (barfoo, .) -. -. - -# Test split user keys containing RANGEDELs. -# Note that this manual compaction is multilevel! - -define -L4 - b.SET.10:b10 -L5 start=b.SET.9 end=b.SET.8 - b.SET.9:b9 - b.SET.8:b8 - b.RANGEDEL.1:z -L5 start=b.SET.7 end=b.SET.6 - b.SET.7:b7 - b.SET.6:b6 - b.RANGEDEL.1:z -L5 start=b.SET.5 end=z.RANGEDEL.72057594037927935 - b.SET.5:b5 - b.SET.4:b4 - b.RANGEDEL.1:z -L6 - b.SET.0:b0 - bat.SET.0:bat - cat.SET.0:cat ----- -4: - 000004:[b#10,SET-b#10,SET] -5: - 000005:[b#9,SET-b#8,SET] - 000006:[b#7,SET-b#6,SET] - 000007:[b#5,SET-z#inf,RANGEDEL] -6: - 000008:[b#0,SET-cat#0,SET] - -compact a-z ----- -6: - 000009:[b#0,SET-b#0,SET] - -iter -first -next ----- -b: (b10, .) -. - -# Ensure an untruncated range tombstone (eg, written by an earlier version of -# Pebble, Cockroach v21.2 or earlier) cannot delete newer keys outside the -# containing file's bounds. -# -# Regression test for cockroachdb/cockroach#89777. - -define target-file-sizes=(1) -L5 start=g.RANGEDEL.1 end=z.RANGEDEL.72057594037927935 - a.RANGEDEL.1:z - m.SET.1:bar -L6 - m.SET.0:bax -L6 start=a.SET.1 end=g.RANGEDEL.72057594037927935 - a.SET.1:foo - a.RANGEDEL.1:z ----- -5: - 000004:[g#1,RANGEDEL-z#inf,RANGEDEL] -6: - 000006:[a#1,SET-g#inf,RANGEDEL] - 000005:[m#0,SET-m#0,SET] - -# Compacting g-z should result in the elision of the range tombstone in the g-z -# span. - -compact g-z ----- -6: - 000006:[a#1,SET-g#inf,RANGEDEL] - 000007:[m#0,SET-m#0,SET] - -# Write a bunch of keys within the keyspace [g,z), and flush them. - -batch -set b b -set h h -set i i -set j j -set k k -set m m -set q q -set y y ----- - -flush ----- -0.0: - 000009:[b#10,SET-b#10,SET] - 000010:[h#11,SET-h#11,SET] - 000011:[i#12,SET-i#12,SET] - 000012:[j#13,SET-j#13,SET] - 000013:[k#14,SET-k#14,SET] - 000014:[m#15,SET-m#15,SET] - 000015:[q#16,SET-q#16,SET] - 000016:[y#17,SET-y#17,SET] -6: - 000006:[a#1,SET-g#inf,RANGEDEL] - 000007:[m#0,SET-m#0,SET] - -# Compact g-z to zero the sequence numbers of the newer keys. - -compact g-z ----- -0.0: - 000009:[b#10,SET-b#10,SET] -6: - 000006:[a#1,SET-g#inf,RANGEDEL] - 000048:[h#0,SET-k#0,SET] - 000049:[m#0,SET-y#0,SET] - -batch -set t t ----- - -flush ----- -0.0: - 000009:[b#10,SET-b#10,SET] - 000051:[t#18,SET-t#18,SET] -6: - 000006:[a#1,SET-g#inf,RANGEDEL] - 000048:[h#0,SET-k#0,SET] - 000049:[m#0,SET-y#0,SET] - -# Compact everything. The batch-committed keys with zeroed sequence numbers (eg, -# h, i, j, k, m, q, y) should all still exist because the a-z tombstone in -# 000006 should be limited in how far it expands. - -compact a-z ----- -6: - 000062:[a#0,SET-i#0,SET] - 000063:[j#0,SET-q#0,SET] - 000064:[t#0,SET-y#0,SET] - -iter -first -next -next -next -next -next -next -next -next -next ----- -a: (foo, .) -b: (b, .) -h: (h, .) -i: (i, .) -j: (j, .) -k: (k, .) -m: (m, .) -q: (q, .) -t: (t, .) -y: (y, .) - -# Ensure an untruncated range tombstone (eg, written by an earlier version of -# Pebble, Cockroach v21.2 or earlier) cannot expand beyond its previous -# truncated bounds. -# -# Regression test for cockroachdb/cockroach#89777. - -define snapshots=(2) -L5 start=g.RANGEDEL.3 end=n.RANGEDEL.72057594037927935 - a.RANGEDEL.3:z - m.SET.1:bar -L5 - s.SET.0:bax -L6 - q.SET.0:foo ----- -5: - 000004:[g#3,RANGEDEL-n#inf,RANGEDEL] - 000005:[s#0,SET-s#0,SET] -6: - 000006:[q#0,SET-q#0,SET] - -# Compacting all the files in a single expansion should not expand the -# untruncated range tombstone to the larger untruncated bounds. The rangedel -# should remain bounded as [a,n). - -compact a-z ----- -6: - 000007:[g#3,RANGEDEL-s#0,SET] diff --git a/testdata/manual_compaction_file_boundaries b/testdata/manual_compaction_file_boundaries deleted file mode 100644 index 1ab48ee3034..00000000000 --- a/testdata/manual_compaction_file_boundaries +++ /dev/null @@ -1,518 +0,0 @@ -# Test the file-size grandparent boundary alignment heuristic. This test sets up -# L3 with a file at each of 'a', 'b', ..., 'z'. It also creates a single file in -# L2 spanning a-z. Then, it commits, flushes and compacts into L1 keys 'a@1', -# 'aa@1', 'ab@1', ..., 'zz@1'. Finally, it tests compacting L1 into L2. -# -# With L3 as the grandparent level, the alignment heuristic should attempt to -# align the output files with grandparent's boundaries. Each output file should -# have a key range formed by the prefix of a single letter. - -define target-file-sizes=(5000, 5000, 5000, 5000) -L2 - a.SET.101: - z.SET.102: -L3 - a.SET.001: -L3 - b.SET.002: -L3 - c.SET.003: -L3 - d.SET.004: -L3 - e.SET.005: -L3 - f.SET.006: -L3 - g.SET.007: -L3 - h.SET.008: -L3 - i.SET.009: -L3 - j.SET.010: -L3 - k.SET.011: -L3 - l.SET.012: -L3 - m.SET.013: -L3 - n.SET.014: -L3 - o.SET.015: -L3 - p.SET.016: -L3 - q.SET.017: -L3 - r.SET.018: -L3 - s.SET.019: -L3 - t.SET.020: -L3 - u.SET.021: -L3 - v.SET.022: -L3 - w.SET.023: -L3 - x.SET.024: -L3 - y.SET.025: -L3 - z.SET.026: ----- -2: - 000004:[a#101,SET-z#102,SET] -3: - 000005:[a#1,SET-a#1,SET] - 000006:[b#2,SET-b#2,SET] - 000007:[c#3,SET-c#3,SET] - 000008:[d#4,SET-d#4,SET] - 000009:[e#5,SET-e#5,SET] - 000010:[f#6,SET-f#6,SET] - 000011:[g#7,SET-g#7,SET] - 000012:[h#8,SET-h#8,SET] - 000013:[i#9,SET-i#9,SET] - 000014:[j#10,SET-j#10,SET] - 000015:[k#11,SET-k#11,SET] - 000016:[l#12,SET-l#12,SET] - 000017:[m#13,SET-m#13,SET] - 000018:[n#14,SET-n#14,SET] - 000019:[o#15,SET-o#15,SET] - 000020:[p#16,SET-p#16,SET] - 000021:[q#17,SET-q#17,SET] - 000022:[r#18,SET-r#18,SET] - 000023:[s#19,SET-s#19,SET] - 000024:[t#20,SET-t#20,SET] - 000025:[u#21,SET-u#21,SET] - 000026:[v#22,SET-v#22,SET] - 000027:[w#23,SET-w#23,SET] - 000028:[x#24,SET-x#24,SET] - 000029:[y#25,SET-y#25,SET] - 000030:[z#26,SET-z#26,SET] - -populate keylen=2 vallen=200 timestamps=(1) ----- -wrote 702 keys - -flush ----- -0.0: - 000033:[a@1#103,SET-aw@1#126,SET] - 000034:[ax@1#127,SET-bt@1#150,SET] - 000035:[bu@1#151,SET-cq@1#174,SET] - 000036:[cr@1#175,SET-dn@1#198,SET] - 000037:[do@1#199,SET-ek@1#222,SET] - 000038:[el@1#223,SET-fh@1#246,SET] - 000039:[fi@1#247,SET-ge@1#270,SET] - 000040:[gf@1#271,SET-hb@1#294,SET] - 000041:[hc@1#295,SET-hz@1#318,SET] - 000042:[i@1#319,SET-iw@1#342,SET] - 000043:[ix@1#343,SET-jt@1#366,SET] - 000044:[ju@1#367,SET-kq@1#390,SET] - 000045:[kr@1#391,SET-ln@1#414,SET] - 000046:[lo@1#415,SET-mk@1#438,SET] - 000047:[ml@1#439,SET-nh@1#462,SET] - 000048:[ni@1#463,SET-oe@1#486,SET] - 000049:[of@1#487,SET-pb@1#510,SET] - 000050:[pc@1#511,SET-pz@1#534,SET] - 000051:[q@1#535,SET-qw@1#558,SET] - 000052:[qx@1#559,SET-rt@1#582,SET] - 000053:[ru@1#583,SET-sq@1#606,SET] - 000054:[sr@1#607,SET-tn@1#630,SET] - 000055:[to@1#631,SET-uk@1#654,SET] - 000056:[ul@1#655,SET-vh@1#678,SET] - 000057:[vi@1#679,SET-we@1#702,SET] - 000058:[wf@1#703,SET-xb@1#726,SET] - 000059:[xc@1#727,SET-xz@1#750,SET] - 000060:[y@1#751,SET-yw@1#774,SET] - 000061:[yx@1#775,SET-zt@1#798,SET] - 000062:[zu@1#799,SET-zz@1#804,SET] -2: - 000004:[a#101,SET-z#102,SET] -3: - 000005:[a#1,SET-a#1,SET] - 000006:[b#2,SET-b#2,SET] - 000007:[c#3,SET-c#3,SET] - 000008:[d#4,SET-d#4,SET] - 000009:[e#5,SET-e#5,SET] - 000010:[f#6,SET-f#6,SET] - 000011:[g#7,SET-g#7,SET] - 000012:[h#8,SET-h#8,SET] - 000013:[i#9,SET-i#9,SET] - 000014:[j#10,SET-j#10,SET] - 000015:[k#11,SET-k#11,SET] - 000016:[l#12,SET-l#12,SET] - 000017:[m#13,SET-m#13,SET] - 000018:[n#14,SET-n#14,SET] - 000019:[o#15,SET-o#15,SET] - 000020:[p#16,SET-p#16,SET] - 000021:[q#17,SET-q#17,SET] - 000022:[r#18,SET-r#18,SET] - 000023:[s#19,SET-s#19,SET] - 000024:[t#20,SET-t#20,SET] - 000025:[u#21,SET-u#21,SET] - 000026:[v#22,SET-v#22,SET] - 000027:[w#23,SET-w#23,SET] - 000028:[x#24,SET-x#24,SET] - 000029:[y#25,SET-y#25,SET] - 000030:[z#26,SET-z#26,SET] - -compact a-zz L0 ----- -1: - 000063:[a@1#103,SET-aw@1#126,SET] - 000064:[ax@1#127,SET-bt@1#150,SET] - 000065:[bu@1#151,SET-cq@1#174,SET] - 000066:[cr@1#175,SET-dn@1#198,SET] - 000067:[do@1#199,SET-ek@1#222,SET] - 000068:[el@1#223,SET-fh@1#246,SET] - 000069:[fi@1#247,SET-ge@1#270,SET] - 000070:[gf@1#271,SET-hb@1#294,SET] - 000071:[hc@1#295,SET-hz@1#318,SET] - 000072:[i@1#319,SET-iw@1#342,SET] - 000073:[ix@1#343,SET-jt@1#366,SET] - 000074:[ju@1#367,SET-kq@1#390,SET] - 000075:[kr@1#391,SET-ln@1#414,SET] - 000076:[lo@1#415,SET-mk@1#438,SET] - 000077:[ml@1#439,SET-nh@1#462,SET] - 000078:[ni@1#463,SET-oe@1#486,SET] - 000079:[of@1#487,SET-pb@1#510,SET] - 000080:[pc@1#511,SET-pz@1#534,SET] - 000081:[q@1#535,SET-qw@1#558,SET] - 000082:[qx@1#559,SET-rt@1#582,SET] - 000083:[ru@1#583,SET-sq@1#606,SET] - 000084:[sr@1#607,SET-tn@1#630,SET] - 000085:[to@1#631,SET-uk@1#654,SET] - 000086:[ul@1#655,SET-vh@1#678,SET] - 000087:[vi@1#679,SET-we@1#702,SET] - 000088:[wf@1#703,SET-xb@1#726,SET] - 000089:[xc@1#727,SET-xz@1#750,SET] - 000090:[y@1#751,SET-yw@1#774,SET] - 000091:[yx@1#775,SET-zt@1#798,SET] - 000092:[zu@1#799,SET-zz@1#804,SET] -2: - 000004:[a#101,SET-z#102,SET] -3: - 000005:[a#1,SET-a#1,SET] - 000006:[b#2,SET-b#2,SET] - 000007:[c#3,SET-c#3,SET] - 000008:[d#4,SET-d#4,SET] - 000009:[e#5,SET-e#5,SET] - 000010:[f#6,SET-f#6,SET] - 000011:[g#7,SET-g#7,SET] - 000012:[h#8,SET-h#8,SET] - 000013:[i#9,SET-i#9,SET] - 000014:[j#10,SET-j#10,SET] - 000015:[k#11,SET-k#11,SET] - 000016:[l#12,SET-l#12,SET] - 000017:[m#13,SET-m#13,SET] - 000018:[n#14,SET-n#14,SET] - 000019:[o#15,SET-o#15,SET] - 000020:[p#16,SET-p#16,SET] - 000021:[q#17,SET-q#17,SET] - 000022:[r#18,SET-r#18,SET] - 000023:[s#19,SET-s#19,SET] - 000024:[t#20,SET-t#20,SET] - 000025:[u#21,SET-u#21,SET] - 000026:[v#22,SET-v#22,SET] - 000027:[w#23,SET-w#23,SET] - 000028:[x#24,SET-x#24,SET] - 000029:[y#25,SET-y#25,SET] - 000030:[z#26,SET-z#26,SET] - -# Perform the actual test. Compacting L1 into L2 should use L3's boundaries to -# inform compaction output splitting. -# -compact a-zz L1 ----- -2: - 000093:[a#101,SET-az@1#129,SET] - 000094:[b@1#130,SET-bz@1#156,SET] - 000095:[c@1#157,SET-cz@1#183,SET] - 000096:[d@1#184,SET-dz@1#210,SET] - 000097:[e@1#211,SET-ez@1#237,SET] - 000098:[f@1#238,SET-fz@1#264,SET] - 000099:[g@1#265,SET-gz@1#291,SET] - 000100:[h@1#292,SET-hz@1#318,SET] - 000101:[i@1#319,SET-iz@1#345,SET] - 000102:[j@1#346,SET-jz@1#372,SET] - 000103:[k@1#373,SET-kz@1#399,SET] - 000104:[l@1#400,SET-lz@1#426,SET] - 000105:[m@1#427,SET-mz@1#453,SET] - 000106:[n@1#454,SET-nz@1#480,SET] - 000107:[o@1#481,SET-oz@1#507,SET] - 000108:[p@1#508,SET-pz@1#534,SET] - 000109:[q@1#535,SET-qz@1#561,SET] - 000110:[r@1#562,SET-rz@1#588,SET] - 000111:[s@1#589,SET-sz@1#615,SET] - 000112:[t@1#616,SET-tz@1#642,SET] - 000113:[u@1#643,SET-uz@1#669,SET] - 000114:[v@1#670,SET-vz@1#696,SET] - 000115:[w@1#697,SET-wz@1#723,SET] - 000116:[x@1#724,SET-xz@1#750,SET] - 000117:[y@1#751,SET-yz@1#777,SET] - 000118:[z#102,SET-zr@1#796,SET] - 000119:[zs@1#797,SET-zz@1#804,SET] -3: - 000005:[a#1,SET-a#1,SET] - 000006:[b#2,SET-b#2,SET] - 000007:[c#3,SET-c#3,SET] - 000008:[d#4,SET-d#4,SET] - 000009:[e#5,SET-e#5,SET] - 000010:[f#6,SET-f#6,SET] - 000011:[g#7,SET-g#7,SET] - 000012:[h#8,SET-h#8,SET] - 000013:[i#9,SET-i#9,SET] - 000014:[j#10,SET-j#10,SET] - 000015:[k#11,SET-k#11,SET] - 000016:[l#12,SET-l#12,SET] - 000017:[m#13,SET-m#13,SET] - 000018:[n#14,SET-n#14,SET] - 000019:[o#15,SET-o#15,SET] - 000020:[p#16,SET-p#16,SET] - 000021:[q#17,SET-q#17,SET] - 000022:[r#18,SET-r#18,SET] - 000023:[s#19,SET-s#19,SET] - 000024:[t#20,SET-t#20,SET] - 000025:[u#21,SET-u#21,SET] - 000026:[v#22,SET-v#22,SET] - 000027:[w#23,SET-w#23,SET] - 000028:[x#24,SET-x#24,SET] - 000029:[y#25,SET-y#25,SET] - 000030:[z#26,SET-z#26,SET] - -file-sizes ----- -L2: - 000093:[a#101,1-az@1#129,1]: 7472 bytes (7.3KB) - 000094:[b@1#130,1-bz@1#156,1]: 6465 bytes (6.3KB) - 000095:[c@1#157,1-cz@1#183,1]: 6465 bytes (6.3KB) - 000096:[d@1#184,1-dz@1#210,1]: 6465 bytes (6.3KB) - 000097:[e@1#211,1-ez@1#237,1]: 6465 bytes (6.3KB) - 000098:[f@1#238,1-fz@1#264,1]: 6465 bytes (6.3KB) - 000099:[g@1#265,1-gz@1#291,1]: 6465 bytes (6.3KB) - 000100:[h@1#292,1-hz@1#318,1]: 6465 bytes (6.3KB) - 000101:[i@1#319,1-iz@1#345,1]: 6465 bytes (6.3KB) - 000102:[j@1#346,1-jz@1#372,1]: 6465 bytes (6.3KB) - 000103:[k@1#373,1-kz@1#399,1]: 6465 bytes (6.3KB) - 000104:[l@1#400,1-lz@1#426,1]: 6465 bytes (6.3KB) - 000105:[m@1#427,1-mz@1#453,1]: 6465 bytes (6.3KB) - 000106:[n@1#454,1-nz@1#480,1]: 6465 bytes (6.3KB) - 000107:[o@1#481,1-oz@1#507,1]: 6465 bytes (6.3KB) - 000108:[p@1#508,1-pz@1#534,1]: 6465 bytes (6.3KB) - 000109:[q@1#535,1-qz@1#561,1]: 6464 bytes (6.3KB) - 000110:[r@1#562,1-rz@1#588,1]: 6465 bytes (6.3KB) - 000111:[s@1#589,1-sz@1#615,1]: 6465 bytes (6.3KB) - 000112:[t@1#616,1-tz@1#642,1]: 6465 bytes (6.3KB) - 000113:[u@1#643,1-uz@1#669,1]: 6465 bytes (6.3KB) - 000114:[v@1#670,1-vz@1#696,1]: 6465 bytes (6.3KB) - 000115:[w@1#697,1-wz@1#723,1]: 6465 bytes (6.3KB) - 000116:[x@1#724,1-xz@1#750,1]: 6465 bytes (6.3KB) - 000117:[y@1#751,1-yz@1#777,1]: 6465 bytes (6.3KB) - 000118:[z#102,1-zr@1#796,1]: 5752 bytes (5.6KB) - 000119:[zs@1#797,1-zz@1#804,1]: 2346 bytes (2.3KB) -L3: - 000005:[a#1,1-a#1,1]: 10638 bytes (10KB) - 000006:[b#2,1-b#2,1]: 10638 bytes (10KB) - 000007:[c#3,1-c#3,1]: 10638 bytes (10KB) - 000008:[d#4,1-d#4,1]: 10638 bytes (10KB) - 000009:[e#5,1-e#5,1]: 10638 bytes (10KB) - 000010:[f#6,1-f#6,1]: 10638 bytes (10KB) - 000011:[g#7,1-g#7,1]: 10638 bytes (10KB) - 000012:[h#8,1-h#8,1]: 10638 bytes (10KB) - 000013:[i#9,1-i#9,1]: 10638 bytes (10KB) - 000014:[j#10,1-j#10,1]: 10638 bytes (10KB) - 000015:[k#11,1-k#11,1]: 10638 bytes (10KB) - 000016:[l#12,1-l#12,1]: 10638 bytes (10KB) - 000017:[m#13,1-m#13,1]: 10638 bytes (10KB) - 000018:[n#14,1-n#14,1]: 10638 bytes (10KB) - 000019:[o#15,1-o#15,1]: 10638 bytes (10KB) - 000020:[p#16,1-p#16,1]: 10638 bytes (10KB) - 000021:[q#17,1-q#17,1]: 10638 bytes (10KB) - 000022:[r#18,1-r#18,1]: 10638 bytes (10KB) - 000023:[s#19,1-s#19,1]: 10638 bytes (10KB) - 000024:[t#20,1-t#20,1]: 10638 bytes (10KB) - 000025:[u#21,1-u#21,1]: 10638 bytes (10KB) - 000026:[v#22,1-v#22,1]: 10638 bytes (10KB) - 000027:[w#23,1-w#23,1]: 10638 bytes (10KB) - 000028:[x#24,1-x#24,1]: 10638 bytes (10KB) - 000029:[y#25,1-y#25,1]: 10638 bytes (10KB) - 000030:[z#26,1-z#26,1]: 10638 bytes (10KB) - -# Test a scenario where there exists a grandparent file (in L3), but the L1->L2 -# compaction doesn't reach it until late in the compaction. The output file -# should be split at 2x the target file size (~10K), despite not being aligned -# with a grandparent. -# -# Additionally, when the compaction does reach the grandparent's start bound, -# the compaction should NOT split the output if the current output is less than -# 0.5x the target file size (~2.5K). -# -# Lastly, once past the final grandparent, the compaction should optimize for -# cutting as close to file size as possible, resulting in an output file ~5K. - -define target-file-sizes=(5000, 5000, 5000, 5000) -L1 - a.SET.201: - b.SET.202: - c.SET.203: - d.SET.204: - e.SET.205: - f.SET.206: - g.SET.207: - h.SET.208: - i.SET.209: - j.SET.210: - k.SET.211: - l.SET.212: - m.SET.213: - n.SET.214: - o.SET.215: -L2 - a.SET.101: - z.SET.102: -L3 - m.SET.001: ----- -1: - 000004:[a#201,SET-o#215,SET] -2: - 000005:[a#101,SET-z#102,SET] -3: - 000006:[m#1,SET-m#1,SET] - -compact a-zz L1 ----- -2: - 000007:[a#201,SET-j#210,SET] - 000008:[k#211,SET-o#215,SET] - 000009:[z#102,SET-z#102,SET] -3: - 000006:[m#1,SET-m#1,SET] - -file-sizes ----- -L2: - 000007:[a#201,1-j#210,1]: 10811 bytes (11KB) - 000008:[k#211,1-o#215,1]: 5723 bytes (5.6KB) - 000009:[z#102,1-z#102,1]: 643 bytes (643B) -L3: - 000006:[m#1,1-m#1,1]: 10638 bytes (10KB) - -# Test the file-size splitter's adaptive tolerance for early-splitting at a -# grandparent boundary. The L1->L2 compaction has many opportunities to split at -# a grandparent boundary at file sizes ≥ 2.5K. Because it's seen more than 8 -# grandparent boundaries, waits until file size is ≥ 90% of the target file size -# (eg, ~4.5K). - -define target-file-sizes=(5000, 5000, 5000, 5000) -L1 - a.SET.201: - b.SET.202: - c.SET.203: - d.SET.204: - e.SET.205: - f.SET.206: - g.SET.207: - h.SET.208: - i.SET.209: - j.SET.210: - k.SET.211: - l.SET.212: - m.SET.213: - n.SET.214: - o.SET.215: -L2 - a.SET.101: - z.SET.102: -L3 - a.SET.001: -L3 - ab.SET.002: -L3 - ac.SET.003: -L3 - ad.SET.004: -L3 - ad.SET.005: -L3 - ad.SET.006: -L3 - ad.SET.007: -L3 - ad.SET.008: -L3 - c.SET.009: -L3 - d.SET.010: -L3 - e.SET.011: -L3 - f.SET.012: -L3 - m.SET.013: ----- -1: - 000004:[a#201,SET-o#215,SET] -2: - 000005:[a#101,SET-z#102,SET] -3: - 000006:[a#1,SET-a#1,SET] - 000007:[ab#2,SET-ab#2,SET] - 000008:[ac#3,SET-ac#3,SET] - 000013:[ad#8,SET-ad#8,SET] - 000012:[ad#7,SET-ad#7,SET] - 000011:[ad#6,SET-ad#6,SET] - 000010:[ad#5,SET-ad#5,SET] - 000009:[ad#4,SET-ad#4,SET] - 000014:[c#9,SET-c#9,SET] - 000015:[d#10,SET-d#10,SET] - 000016:[e#11,SET-e#11,SET] - 000017:[f#12,SET-f#12,SET] - 000018:[m#13,SET-m#13,SET] - -compact a-zz L1 ----- -2: - 000019:[a#201,SET-e#205,SET] - 000020:[f#206,SET-l#212,SET] - 000021:[m#213,SET-z#102,SET] -3: - 000006:[a#1,SET-a#1,SET] - 000007:[ab#2,SET-ab#2,SET] - 000008:[ac#3,SET-ac#3,SET] - 000013:[ad#8,SET-ad#8,SET] - 000012:[ad#7,SET-ad#7,SET] - 000011:[ad#6,SET-ad#6,SET] - 000010:[ad#5,SET-ad#5,SET] - 000009:[ad#4,SET-ad#4,SET] - 000014:[c#9,SET-c#9,SET] - 000015:[d#10,SET-d#10,SET] - 000016:[e#11,SET-e#11,SET] - 000017:[f#12,SET-f#12,SET] - 000018:[m#13,SET-m#13,SET] - -file-sizes ----- -L2: - 000019:[a#201,1-e#205,1]: 5723 bytes (5.6KB) - 000020:[f#206,1-l#212,1]: 7749 bytes (7.6KB) - 000021:[m#213,1-z#102,1]: 3686 bytes (3.6KB) -L3: - 000006:[a#1,1-a#1,1]: 1638 bytes (1.6KB) - 000007:[ab#2,1-ab#2,1]: 1639 bytes (1.6KB) - 000008:[ac#3,1-ac#3,1]: 1639 bytes (1.6KB) - 000013:[ad#8,1-ad#8,1]: 1639 bytes (1.6KB) - 000012:[ad#7,1-ad#7,1]: 1639 bytes (1.6KB) - 000011:[ad#6,1-ad#6,1]: 1639 bytes (1.6KB) - 000010:[ad#5,1-ad#5,1]: 1639 bytes (1.6KB) - 000009:[ad#4,1-ad#4,1]: 1639 bytes (1.6KB) - 000014:[c#9,1-c#9,1]: 1638 bytes (1.6KB) - 000015:[d#10,1-d#10,1]: 1638 bytes (1.6KB) - 000016:[e#11,1-e#11,1]: 1638 bytes (1.6KB) - 000017:[f#12,1-f#12,1]: 1638 bytes (1.6KB) - 000018:[m#13,1-m#13,1]: 1638 bytes (1.6KB) diff --git a/testdata/manual_compaction_set_with_del b/testdata/manual_compaction_set_with_del deleted file mode 100644 index 95dc46cf94d..00000000000 --- a/testdata/manual_compaction_set_with_del +++ /dev/null @@ -1,863 +0,0 @@ -batch -set a 1 -set b 2 ----- - -compact a-b ----- -6: - 000005:[a#10,SET-b#11,SET] - -batch -set c 3 -set d 4 ----- - -compact c-d ----- -6: - 000005:[a#10,SET-b#11,SET] - 000007:[c#12,SET-d#13,SET] - -batch -set b 5 -set c 6 ----- - -compact a-d ----- -6: - 000010:[a#0,SET-d#0,SET] - -# This also tests flushing a memtable that only contains range -# deletions. - -batch -del-range a e ----- - -compact a-d ----- - -# Test that a multi-output-file compaction generates non-overlapping files. - -define target-file-sizes=(100, 1) -L0 - b.SET.1:v -L0 - a.SET.2:v ----- -0.0: - 000005:[a#2,SET-a#2,SET] - 000004:[b#1,SET-b#1,SET] - -compact a-b ----- -1: - 000006:[a#0,SET-a#0,SET] - 000007:[b#0,SET-b#0,SET] - -# A range tombstone extends past the grandparent file boundary used to limit the -# size of future compactions. Verify the range tombstone is split at that file -# boundary. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.SET.3:v -L2 - a.RANGEDEL.2:e -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v - d.SET.0:v ----- -1: - 000004:[a#3,SET-a#3,SET] -2: - 000005:[a#2,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - -wait-pending-table-stats -000005 ----- -num-entries: 1 -num-deletions: 1 -num-range-key-sets: 0 -point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1278 - -compact a-e L1 ----- -2: - 000008:[a#3,SETWITHDEL-c#inf,RANGEDEL] - 000009:[c#2,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - -wait-pending-table-stats -000008 ----- -num-entries: 2 -num-deletions: 1 -num-range-key-sets: 0 -point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 639 - -# Same as above, except range tombstone covers multiple grandparent file boundaries. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.SET.3:v -L2 - a.RANGEDEL.2:g -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v - d.SET.0:v -L3 - e.SET.0:v - f.SET.1:v -L3 - f.SET.0:v - g.SET.0:v ----- -1: - 000004:[a#3,SET-a#3,SET] -2: - 000005:[a#2,RANGEDEL-g#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - 000008:[e#0,SET-f#1,SET] - 000009:[f#0,SET-g#0,SET] - -compact a-e L1 ----- -2: - 000010:[a#3,SETWITHDEL-c#inf,RANGEDEL] - 000011:[c#2,RANGEDEL-e#inf,RANGEDEL] - 000012:[e#2,RANGEDEL-f#inf,RANGEDEL] - 000013:[f#2,RANGEDEL-g#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - 000008:[e#0,SET-f#1,SET] - 000009:[f#0,SET-g#0,SET] - -# A range tombstone covers multiple grandparent file boundaries between point keys, -# rather than after all point keys. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.SET.3:v - h.SET.3:v -L2 - a.RANGEDEL.2:g -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v - d.SET.0:v -L3 - e.SET.0:v - f.SET.1:v ----- -1: - 000004:[a#3,SET-h#3,SET] -2: - 000005:[a#2,RANGEDEL-g#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - 000008:[e#0,SET-f#1,SET] - -compact a-e L1 ----- -2: - 000009:[a#3,SETWITHDEL-c#inf,RANGEDEL] - 000010:[c#2,RANGEDEL-h#3,SET] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - 000008:[e#0,SET-f#1,SET] - -# A range tombstone is the first and only item output by a compaction, and it -# extends past the grandparent file boundary used to limit the size of future -# compactions. Verify the range tombstone is split at that file boundary. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.RANGEDEL.3:e -L2 - a.SET.2:v -L3 - a.SET.0:v - b.SET.0:v -L3 - c.SET.0:v - d.SET.0:v ----- -1: - 000004:[a#3,RANGEDEL-e#inf,RANGEDEL] -2: - 000005:[a#2,SET-a#2,SET] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - -compact a-e L1 ----- -2: - 000008:[a#3,RANGEDEL-c#inf,RANGEDEL] - 000009:[c#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[a#0,SET-b#0,SET] - 000007:[c#0,SET-d#0,SET] - -# An elided range tombstone is the first item encountered by a compaction, -# and the grandparent limit set by it extends to the next item, also a range -# tombstone. The first item should be elided, and the second item should -# reset the grandparent limit. - -define target-file-sizes=(100, 100, 100, 100) -L1 - a.RANGEDEL.4:d -L1 - grandparent.RANGEDEL.2:z - h.SET.3:v -L2 - grandparent.SET.1:v -L3 - grandparent.SET.0:v -L3 - m.SET.0:v ----- -1: - 000004:[a#4,RANGEDEL-d#inf,RANGEDEL] - 000005:[grandparent#2,RANGEDEL-z#inf,RANGEDEL] -2: - 000006:[grandparent#1,SET-grandparent#1,SET] -3: - 000007:[grandparent#0,SET-grandparent#0,SET] - 000008:[m#0,SET-m#0,SET] - -compact a-h L1 ----- -2: - 000009:[grandparent#2,RANGEDEL-m#inf,RANGEDEL] - 000010:[m#2,RANGEDEL-z#inf,RANGEDEL] -3: - 000007:[grandparent#0,SET-grandparent#0,SET] - 000008:[m#0,SET-m#0,SET] - -# Setup such that grandparent overlap limit is exceeded multiple times at the same user key ("b"). -# Ensures the compaction output files are non-overlapping. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.SET.2:v - c.SET.2:v -L2 - a.RANGEDEL.3:c -L3 - b.SET.2:v -L3 - b.SET.1:v -L3 - b.SET.0:v ----- -1: - 000004:[a#2,SET-c#2,SET] -2: - 000005:[a#3,RANGEDEL-c#inf,RANGEDEL] -3: - 000006:[b#2,SET-b#2,SET] - 000007:[b#1,SET-b#1,SET] - 000008:[b#0,SET-b#0,SET] - -compact a-c L1 ----- -2: - 000009:[a#3,RANGEDEL-b#inf,RANGEDEL] - 000010:[b#3,RANGEDEL-c#2,SET] -3: - 000006:[b#2,SET-b#2,SET] - 000007:[b#1,SET-b#1,SET] - 000008:[b#0,SET-b#0,SET] - -# Regression test for a bug where compaction would stop process range -# tombstones for an input level upon finding an sstable in the input -# level with no range tombstones. In the scenario below, sstable 6 -# does not contain any range tombstones while sstable 7 does. Both are -# compacted together with sstable 5. - -reset ----- - -batch -set a 1 -set b 1 -set c 1 -set d 1 -set z 1 ----- - -compact a-z ----- -6: - 000005:[a#10,SET-z#14,SET] - -build ext1 -set a 2 ----- - -build ext2 -set b 2 -del-range c z ----- - -ingest ext1 ext2 ----- -0.0: - 000006:[a#15,SET-a#15,SET] - 000007:[b#16,SET-z#inf,RANGEDEL] -6: - 000005:[a#10,SET-z#14,SET] - -iter -first -next -next -next ----- -a: (2, .) -b: (2, .) -z: (1, .) -. - -compact a-z ----- -6: - 000008:[a#0,SET-z#0,SET] - -iter -first -next -next -next ----- -a: (2, .) -b: (2, .) -z: (1, .) -. - -# Regresion test for a bug in sstable smallest boundary generation -# where the smallest key for an sstable was set to a key "larger" than -# the start key of the first range tombstone. This in turn fouled up -# the processing logic of range tombstones used by mergingIter which -# allowed stepping out of an sstable even though it contained a range -# tombstone that covered keys in lower levels. - -define target-file-sizes=(1, 1, 1, 1) -L0 - c.SET.4:4 -L1 - a.SET.3:3 -L2 - a.RANGEDEL.2:e -L3 - b.SET.1:1 ----- -0.0: - 000004:[c#4,SET-c#4,SET] -1: - 000005:[a#3,SET-a#3,SET] -2: - 000006:[a#2,RANGEDEL-e#inf,RANGEDEL] -3: - 000007:[b#1,SET-b#1,SET] - -compact a-e L1 ----- -0.0: - 000004:[c#4,SET-c#4,SET] -2: - 000008:[a#3,SETWITHDEL-b#inf,RANGEDEL] - 000009:[b#2,RANGEDEL-e#inf,RANGEDEL] -3: - 000007:[b#1,SET-b#1,SET] - -# We should only see a:3 and c:4 at this point. - -iter -first -next -next ----- -a: (3, .) -c: (4, .) -. - -# The bug allowed seeing b:1 during reverse iteration. - -iter -last -prev -prev ----- -c: (4, .) -a: (3, .) -. - -# This is a similar scenario to the one above. In older versions of Pebble this -# case necessitated adjusting the seqnum of the range tombstone to -# prev.LargestKey.SeqNum-1. We no longer allow user keys to be split across -# sstables, and the seqnum adjustment is no longer necessary. -# -# Note the target-file-size of 26 is specially tailored to get the -# desired compaction output. - -define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) -L1 - a.SET.4:4 -L1 - b.SET.2:2 - b.RANGEDEL.3:e -L3 - b.SET.1:1 ----- -1: - 000004:[a#4,SET-a#4,SET] - 000005:[b#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -compact a-e L1 ----- -2: - 000007:[a#4,SET-a#4,SET] - 000008:[b#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -iter -first -next -last -prev ----- -a: (4, .) -. -a: (4, .) -. - -# Similar to the preceding scenario, except the range tombstone has -# the same seqnum as the largest key in the preceding file. - -define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) -L1 - a.SET.4:4 -L1 - b.SET.3:3 - b.RANGEDEL.3:e -L3 - b.SET.1:1 ----- -1: - 000004:[a#4,SET-a#4,SET] - 000005:[b#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -compact a-e L1 ----- -2: - 000007:[a#4,SET-a#4,SET] - 000008:[b#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -iter -first -next -next -last -prev -prev ----- -a: (4, .) -b: (3, .) -. -b: (3, .) -a: (4, .) -. - -# Similar to the preceding scenario, except the range tombstone has -# a smaller seqnum than the largest key in the preceding file. - -define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) -L1 - a.SET.4:4 -L1 - b.SET.4:4 - b.RANGEDEL.2:e -L3 - b.SET.1:1 ----- -1: - 000004:[a#4,SET-a#4,SET] - 000005:[b#4,SET-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -compact a-e L1 ----- -2: - 000007:[a#4,SET-a#4,SET] - 000008:[b#4,SET-e#inf,RANGEDEL] -3: - 000006:[b#1,SET-b#1,SET] - -iter -first -next -next -last -prev -prev ----- -a: (4, .) -b: (4, .) -. -b: (4, .) -a: (4, .) -. - -# Test a scenario where the last point key in an sstable has a seqnum -# of 0. - -define target-file-sizes=(1, 1, 26) snapshots=(2) -L1 - a.SET.3:3 - b.RANGEDEL.3:e - b.SET.0:0 -L3 - a.RANGEDEL.2:b -L3 - c.SET.0:0 - d.SET.0:0 ----- -1: - 000004:[a#3,SET-e#inf,RANGEDEL] -3: - 000005:[a#2,RANGEDEL-b#inf,RANGEDEL] - 000006:[c#0,SET-d#0,SET] - -iter -last -prev ----- -a: (3, .) -. - -compact a-e L1 ----- -2: - 000007:[a#3,SET-c#inf,RANGEDEL] - 000008:[c#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000005:[a#2,RANGEDEL-b#inf,RANGEDEL] - 000006:[c#0,SET-d#0,SET] - -iter -last -prev ----- -a: (3, .) -. - -# Test a scenario where the last point key in an sstable before the -# grandparent limit is reached has a seqnum of 0. We want to cut the -# sstable after the next point key is added, rather than continuing to -# add keys indefinitely (or till the size limit is reached). - -define target-file-sizes=(100, 1, 52) snapshots=(2) -L1 - a.SET.3:3 - b.RANGEDEL.3:e - b.SET.0:0 - c.SET.3:1 - d.SET.1:1 -L3 - c.RANGEDEL.2:d ----- -1: - 000004:[a#3,SET-e#inf,RANGEDEL] -3: - 000005:[c#2,RANGEDEL-d#inf,RANGEDEL] - -compact a-f L1 ----- -2: - 000006:[a#3,SET-c#inf,RANGEDEL] - 000007:[c#3,RANGEDEL-e#inf,RANGEDEL] -3: - 000005:[c#2,RANGEDEL-d#inf,RANGEDEL] - - -# Test a scenario where we the last point key in an sstable has a -# seqnum of 0, but there is another range tombstone later in the -# compaction. This scenario was previously triggering an assertion due -# to the rangedel.Fragmenter being finished prematurely. - -define target-file-sizes=(1, 1, 1) -L1 - a.SET.0:0 - c.RANGEDEL.1:d -L3 - b.SET.0:0 ----- -1: - 000004:[a#0,SET-d#inf,RANGEDEL] -3: - 000005:[b#0,SET-b#0,SET] - -compact a-e L1 ----- -2: - 000006:[a#0,SET-a#0,SET] -3: - 000005:[b#0,SET-b#0,SET] - -define target-file-sizes=(1, 1, 1, 1) -L0 - b.SET.1:v -L0 - a.SET.2:v ----- -0.0: - 000005:[a#2,SET-a#2,SET] - 000004:[b#1,SET-b#1,SET] - -add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z ----- - -async-compact a-b L0 ----- -manual compaction blocked until ongoing finished -1: - 000006:[a#0,SET-a#0,SET] - 000007:[b#0,SET-b#0,SET] - -compact a-b L1 ----- -2: - 000008:[a#0,SET-a#0,SET] - 000009:[b#0,SET-b#0,SET] - -add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z ----- - -async-compact a-b L2 ----- -manual compaction blocked until ongoing finished -3: - 000010:[a#0,SET-a#0,SET] - 000011:[b#0,SET-b#0,SET] - -add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z ----- - -set-concurrent-compactions num=2 ----- - -async-compact a-b L3 ----- -manual compaction did not block for ongoing -4: - 000012:[a#0,SET-a#0,SET] - 000013:[b#0,SET-b#0,SET] - -remove-ongoing-compaction ----- - -add-ongoing-compaction startLevel=4 outputLevel=5 start=a end=b ----- - -async-compact a-b L4 ----- -manual compaction blocked until ongoing finished -5: - 000014:[a#0,SET-a#0,SET] - 000015:[b#0,SET-b#0,SET] - -# Test of a scenario where consecutive elided range tombstones and grandparent -# boundaries could result in an invariant violation in the rangedel fragmenter. - -define target-file-sizes=(1, 1, 1, 1) -L1 - a.RANGEDEL.4:b - c.RANGEDEL.4:d - e.RANGEDEL.4:f -L1 - g.RANGEDEL.6:h - i.RANGEDEL.4:j -L1 - k.RANGEDEL.5:q - m.RANGEDEL.4:q -L2 - a.SET.2:foo -L3 - a.SET.1:foo - c.SET.1:foo -L3 - ff.SET.1:v -L3 - k.SET.1:foo ----- -1: - 000004:[a#4,RANGEDEL-f#inf,RANGEDEL] - 000005:[g#6,RANGEDEL-j#inf,RANGEDEL] - 000006:[k#5,RANGEDEL-q#inf,RANGEDEL] -2: - 000007:[a#2,SET-a#2,SET] -3: - 000008:[a#1,SET-c#1,SET] - 000009:[ff#1,SET-ff#1,SET] - 000010:[k#1,SET-k#1,SET] - -compact a-q L1 ----- -2: - 000011:[a#4,RANGEDEL-d#inf,RANGEDEL] - 000012:[k#5,RANGEDEL-m#inf,RANGEDEL] -3: - 000008:[a#1,SET-c#1,SET] - 000009:[ff#1,SET-ff#1,SET] - 000010:[k#1,SET-k#1,SET] - -# Test a case where a new output file is started, there are no previous output -# files, there are no additional keys (key = nil) and the rangedel fragmenter -# is non-empty. -define target-file-sizes=(1, 1, 1) -L1 - a.RANGEDEL.10:b - d.RANGEDEL.9:e - q.RANGEDEL.8:r -L2 - g.RANGEDEL.7:h -L3 - q.SET.6:6 ----- -1: - 000004:[a#10,RANGEDEL-r#inf,RANGEDEL] -2: - 000005:[g#7,RANGEDEL-h#inf,RANGEDEL] -3: - 000006:[q#6,SET-q#6,SET] - -compact a-r L1 ----- -2: - 000007:[q#8,RANGEDEL-r#inf,RANGEDEL] -3: - 000006:[q#6,SET-q#6,SET] - -define target-file-sizes=(100, 100, 100) -L1 - a.RANGEDEL.10:b - b.SET.0:foo - d.RANGEDEL.0:e - j.SET.10:foo -L2 - f.RANGEDEL.7:g -L3 - c.SET.6:6 -L3 - c.SET.5:5 -L3 - c.SET.4:4 -L4 - a.SET.0:0 - f.SET.0:0 ----- -1: - 000004:[a#10,RANGEDEL-j#10,SET] -2: - 000005:[f#7,RANGEDEL-g#inf,RANGEDEL] -3: - 000006:[c#6,SET-c#6,SET] - 000007:[c#5,SET-c#5,SET] - 000008:[c#4,SET-c#4,SET] -4: - 000009:[a#0,SET-f#0,SET] - -compact a-r L1 ----- -2: - 000010:[a#10,RANGEDEL-b#0,SET] - 000011:[d#0,RANGEDEL-j#10,SET] -3: - 000006:[c#6,SET-c#6,SET] - 000007:[c#5,SET-c#5,SET] - 000008:[c#4,SET-c#4,SET] -4: - 000009:[a#0,SET-f#0,SET] - -# Test a snapshot that separates a range deletion from all the data that it -# deletes. Ensure that we respect the target-file-size and split into multiple -# outputs. - -define target-file-sizes=(1, 1, 1) snapshots=(14) -L1 - a.RANGEDEL.15:z - b.SET.11:foo - c.SET.11:foo -L2 - c.SET.0:foo - d.SET.0:foo ----- -1: - 000004:[a#15,RANGEDEL-z#inf,RANGEDEL] -2: - 000005:[c#0,SET-d#0,SET] - -compact a-z L1 ----- -2: - 000006:[a#15,RANGEDEL-c#inf,RANGEDEL] - 000007:[c#15,RANGEDEL-d#inf,RANGEDEL] - 000008:[d#15,RANGEDEL-z#inf,RANGEDEL] - -# Test an interaction between a range deletion that will be elided with -# output splitting. Ensure that the output is still split (previous versions -# of the code did not, because of intricacies around preventing a zero -# sequence number in an output's largest key). - -define target-file-sizes=(1, 1, 1) -L1 - a.RANGEDEL.10:z - b.SET.11:foo - c.SET.11:foo -L2 - c.SET.0:foo - d.SET.0:foo ----- -1: - 000004:[a#10,RANGEDEL-z#inf,RANGEDEL] -2: - 000005:[c#0,SET-d#0,SET] - -compact a-z L1 ----- -2: - 000006:[b#0,SET-b#0,SET] - 000007:[c#0,SET-c#0,SET] diff --git a/testdata/range_del b/testdata/range_del index 14102e07bbc..53cca9d90e9 100644 --- a/testdata/range_del +++ b/testdata/range_del @@ -629,166 +629,6 @@ b: (4, .) a: (4, .) . -# User-key that spans tables in a level. - -define -L1 - a.SET.12:3 -L1 - a.SET.11:2 -L1 - a.SET.10:1 ----- -mem: 1 -1: - 000004:[a#12,SET-a#12,SET] - 000005:[a#11,SET-a#11,SET] - 000006:[a#10,SET-a#10,SET] - -get seq=10 -a ----- -a: pebble: not found - -get seq=11 -a ----- -a:1 - -get seq=12 -a ----- -a:2 - -get seq=13 -a ----- -a:3 - -iter seq=11 -first -seek-ge a -seek-ge b -last -seek-lt a -seek-lt b ----- -a: (1, .) -a: (1, .) -. -a: (1, .) -. -a: (1, .) - -iter seq=12 -first -seek-ge a -seek-ge b -last -seek-lt a -seek-lt b ----- -a: (2, .) -a: (2, .) -. -a: (2, .) -. -a: (2, .) - -iter seq=13 -first -seek-ge a -seek-ge b -last -seek-lt a -seek-lt b ----- -a: (3, .) -a: (3, .) -. -a: (3, .) -. -a: (3, .) - -define -L1 - a.MERGE.12:3 -L1 - a.MERGE.11:2 -L1 - a.MERGE.10:1 ----- -mem: 1 -1: - 000004:[a#12,MERGE-a#12,MERGE] - 000005:[a#11,MERGE-a#11,MERGE] - 000006:[a#10,MERGE-a#10,MERGE] - -get seq=10 -a ----- -a: pebble: not found - -get seq=11 -a ----- -a:1 - -get seq=12 -a ----- -a:12 - -get seq=13 -a ----- -a:123 - -iter seq=11 -first -seek-ge a -seek-ge b -last -seek-lt a -seek-lt b ----- -a: (1, .) -a: (1, .) -. -a: (1, .) -. -a: (1, .) - -iter seq=12 -first -seek-ge a -seek-ge b -last -seek-lt a -seek-lt b ----- -a: (12, .) -a: (12, .) -. -a: (12, .) -. -a: (12, .) - -iter seq=13 -first -seek-ge a -seek-ge b -last -seek-lt a -seek-lt b ----- -a: (123, .) -a: (123, .) -. -a: (123, .) -. -a: (123, .) - # User-key spread across multiple levels. define @@ -1171,13 +1011,13 @@ mem: 1 compact a-e ---- 1: - 000007:[a#11,SET-c#inf,RANGEDEL] + 000007:[a#11,SETWITHDEL-c#inf,RANGEDEL] 000008:[c#12,SET-e#inf,RANGEDEL] compact d-e ---- 1: - 000007:[a#11,SET-c#inf,RANGEDEL] + 000007:[a#11,SETWITHDEL-c#inf,RANGEDEL] 2: 000008:[c#12,SET-e#inf,RANGEDEL] @@ -1209,7 +1049,7 @@ mem: 1 compact a-e ---- 1: - 000007:[a#11,SET-c#inf,RANGEDEL] + 000007:[a#11,SETWITHDEL-c#inf,RANGEDEL] 000008:[c#12,SET-e#inf,RANGEDEL] compact a-b @@ -1217,7 +1057,7 @@ compact a-b 1: 000008:[c#12,SET-e#inf,RANGEDEL] 2: - 000007:[a#11,SET-c#inf,RANGEDEL] + 000007:[a#11,SETWITHDEL-c#inf,RANGEDEL] iter seq=13 seek-lt d @@ -1257,7 +1097,7 @@ mem: 1 compact a-b ---- 1: - 000008:[a#11,SET-c#inf,RANGEDEL] + 000008:[a#11,SETWITHDEL-c#inf,RANGEDEL] 000009:[c#12,SET-d#inf,RANGEDEL] 000010:[d#10,RANGEDEL-e#inf,RANGEDEL] 2: @@ -1266,7 +1106,7 @@ compact a-b compact d-e ---- 1: - 000008:[a#11,SET-c#inf,RANGEDEL] + 000008:[a#11,SETWITHDEL-c#inf,RANGEDEL] 000009:[c#12,SET-d#inf,RANGEDEL] 3: 000011:[d#10,RANGEDEL-e#inf,RANGEDEL] @@ -1281,7 +1121,7 @@ compact a-b L1 1: 000009:[c#12,SET-d#inf,RANGEDEL] 2: - 000008:[a#11,SET-c#inf,RANGEDEL] + 000008:[a#11,SETWITHDEL-c#inf,RANGEDEL] 3: 000011:[d#10,RANGEDEL-e#inf,RANGEDEL] @@ -1320,7 +1160,7 @@ compact a-b 0.0: 000007:[f#13,SET-f#13,SET] 1: - 000009:[a#11,SET-c#inf,RANGEDEL] + 000009:[a#11,SETWITHDEL-c#inf,RANGEDEL] 000010:[c#12,SET-d#inf,RANGEDEL] 000011:[d#10,RANGEDEL-e#inf,RANGEDEL] 2: @@ -1331,7 +1171,7 @@ compact d-e 0.0: 000007:[f#13,SET-f#13,SET] 1: - 000009:[a#11,SET-c#inf,RANGEDEL] + 000009:[a#11,SETWITHDEL-c#inf,RANGEDEL] 000010:[c#12,SET-d#inf,RANGEDEL] 3: 000012:[d#10,RANGEDEL-e#inf,RANGEDEL] @@ -1344,7 +1184,7 @@ c:v compact f-f L0 ---- 1: - 000009:[a#11,SET-c#inf,RANGEDEL] + 000009:[a#11,SETWITHDEL-c#inf,RANGEDEL] 000010:[c#12,SET-d#inf,RANGEDEL] 000007:[f#13,SET-f#13,SET] 3: @@ -1353,8 +1193,8 @@ compact f-f L0 compact a-f L1 ---- 2: - 000013:[a#11,SET-c#inf,RANGEDEL] - 000014:[c#12,SET-d#inf,RANGEDEL] + 000013:[a#11,SETWITHDEL-c#inf,RANGEDEL] + 000014:[c#12,SETWITHDEL-d#inf,RANGEDEL] 000015:[f#13,SET-f#13,SET] 3: 000012:[d#10,RANGEDEL-e#inf,RANGEDEL] @@ -1401,7 +1241,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 836 +range-deletions-bytes-estimate: 699 wait-pending-table-stats 000004 @@ -1410,7 +1250,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1672 +range-deletions-bytes-estimate: 1398 wait-pending-table-stats 000005 @@ -1419,7 +1259,7 @@ num-entries: 2 num-deletions: 2 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1672 +range-deletions-bytes-estimate: 1398 # Range deletions with varying overlap. @@ -1522,7 +1362,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 782 +range-deletions-bytes-estimate: 645 wait-pending-table-stats 000006 @@ -1531,7 +1371,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 771 +range-deletions-bytes-estimate: 634 wait-pending-table-stats 000004 @@ -1540,4 +1380,4 @@ num-entries: 2 num-deletions: 2 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1553 +range-deletions-bytes-estimate: 1279 diff --git a/testdata/rocksdb-ingest-only/000003.log b/testdata/rocksdb-ingest-only/000003.log deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testdata/rocksdb-ingest-only/000006.sst b/testdata/rocksdb-ingest-only/000006.sst deleted file mode 100644 index 1b98083a748..00000000000 Binary files a/testdata/rocksdb-ingest-only/000006.sst and /dev/null differ diff --git a/testdata/rocksdb-ingest-only/CURRENT b/testdata/rocksdb-ingest-only/CURRENT deleted file mode 100644 index 875cf233554..00000000000 --- a/testdata/rocksdb-ingest-only/CURRENT +++ /dev/null @@ -1 +0,0 @@ -MANIFEST-000007 diff --git a/testdata/rocksdb-ingest-only/IDENTITY b/testdata/rocksdb-ingest-only/IDENTITY deleted file mode 100644 index b11f8d2ce17..00000000000 --- a/testdata/rocksdb-ingest-only/IDENTITY +++ /dev/null @@ -1 +0,0 @@ -160040b17ab41758-6d6a810cd585edc2 \ No newline at end of file diff --git a/testdata/rocksdb-ingest-only/LOCK b/testdata/rocksdb-ingest-only/LOCK deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testdata/rocksdb-ingest-only/MANIFEST-000001 b/testdata/rocksdb-ingest-only/MANIFEST-000001 deleted file mode 100644 index d89025ae6eb..00000000000 Binary files a/testdata/rocksdb-ingest-only/MANIFEST-000001 and /dev/null differ diff --git a/testdata/rocksdb-ingest-only/MANIFEST-000007 b/testdata/rocksdb-ingest-only/MANIFEST-000007 deleted file mode 100644 index 4b500e23263..00000000000 Binary files a/testdata/rocksdb-ingest-only/MANIFEST-000007 and /dev/null differ diff --git a/testdata/rocksdb-ingest-only/OPTIONS-000005 b/testdata/rocksdb-ingest-only/OPTIONS-000005 deleted file mode 100644 index 542dea6e6d6..00000000000 --- a/testdata/rocksdb-ingest-only/OPTIONS-000005 +++ /dev/null @@ -1,158 +0,0 @@ -# This is a RocksDB option file. -# -# For detailed file format spec, please refer to the example file -# in examples/rocksdb_option_file_example.ini -# - -[Version] - rocksdb_version=6.2.1 - options_file_version=1.1 - -[DBOptions] - allow_mmap_writes=false - base_background_compactions=-1 - new_table_reader_for_compaction_inputs=false - db_log_dir= - wal_recovery_mode=kTolerateCorruptedTailRecords - use_direct_reads=false - write_thread_max_yield_usec=100 - max_manifest_file_size=134217728 - allow_2pc=false - allow_fallocate=true - avoid_unnecessary_blocking_io=false - fail_if_options_file_error=false - allow_ingest_behind=false - allow_mmap_reads=false - skip_log_error_on_recovery=false - strict_bytes_per_sync=true - stats_history_buffer_size=1048576 - recycle_log_file_num=1 - delete_obsolete_files_period_micros=21600000000 - compaction_readahead_size=0 - use_direct_io_for_flush_and_compaction=false - log_file_time_to_roll=0 - create_missing_column_families=false - advise_random_on_open=true - max_log_file_size=0 - stats_dump_period_sec=0 - enable_thread_tracking=false - use_adaptive_mutex=false - create_if_missing=true - is_fd_close_on_exec=true - max_background_flushes=-1 - manifest_preallocation_size=4194304 - error_if_exists=false - skip_stats_update_on_db_open=false - max_open_files=256 - random_access_max_buffer_size=1048576 - use_fsync=false - max_background_jobs=4 - two_write_queues=false - max_background_compactions=-1 - max_file_opening_threads=16 - table_cache_numshardbits=6 - keep_log_file_num=1000 - avoid_flush_during_shutdown=false - db_write_buffer_size=0 - max_total_wal_size=0 - wal_dir=/var/folders/df/d5bj8zws5wvbhc_6szqzmh6r0000gn/T/TestRocksPebbleCheck_rocksdb,pebble321860949/store - max_subcompactions=1 - atomic_flush=false - WAL_size_limit_MB=0 - paranoid_checks=true - allow_concurrent_memtable_write=true - writable_file_max_buffer_size=1048576 - WAL_ttl_seconds=0 - delayed_write_rate=16777216 - bytes_per_sync=524288 - wal_bytes_per_sync=0 - stats_persist_period_sec=600 - preserve_deletes=false - enable_pipelined_write=false - enable_write_thread_adaptive_yield=true - write_thread_slow_yield_usec=3 - access_hint_on_compaction_start=NORMAL - info_log_level=INFO_LEVEL - dump_malloc_stats=false - avoid_flush_during_recovery=false - manual_wal_flush=true - - -[CFOptions "default"] - report_bg_io_stats=false - inplace_update_support=false - max_compaction_bytes=104857600 - disable_auto_compactions=false - write_buffer_size=67108864 - bloom_locality=0 - max_bytes_for_level_multiplier=10.000000 - compaction_filter_factory=nullptr - optimize_filters_for_hits=true - target_file_size_base=4194304 - max_write_buffer_number_to_maintain=0 - hard_pending_compaction_bytes_limit=4400193994752 - paranoid_file_checks=false - memtable_prefix_bloom_size_ratio=0.000000 - force_consistency_checks=false - max_write_buffer_number=4 - ttl=0 - max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 - level0_slowdown_writes_trigger=950 - level_compaction_dynamic_level_bytes=true - compaction_options_fifo={allow_compaction=false;max_table_files_size=1073741824;} - inplace_update_num_locks=10000 - level0_file_num_compaction_trigger=2 - compression=kSnappyCompression - level0_stop_writes_trigger=1000 - periodic_compaction_seconds=0 - num_levels=7 - table_factory=BlockBasedTable - target_file_size_multiplier=2 - min_write_buffer_number_to_merge=1 - arena_block_size=8388608 - max_successive_merges=0 - memtable_huge_page_size=0 - compaction_pri=kMinOverlappingRatio - soft_pending_compaction_bytes_limit=2199023255552 - memtable_whole_key_filtering=false - memtable_insert_with_hint_prefix_extractor=nullptr - max_bytes_for_level_base=67108864 - comparator=cockroach_comparator - max_sequential_skip_in_iterations=8 - compression_per_level= - bottommost_compression=kDisableCompressionOption - prefix_extractor=cockroach_prefix_extractor - memtable_factory=SkipListFactory - compaction_filter=nullptr - compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;} - merge_operator=cockroach_merge_operator - compaction_style=kCompactionStyleLevel - sample_for_compression=0 - -[TableOptions/BlockBasedTable "default"] - read_amp_bytes_per_bit=8589934592 - format_version=2 - verify_compression=false - partition_filters=false - enable_index_compression=true - checksum=kCRC32c - index_block_restart_interval=1 - pin_top_level_index_and_filter=true - block_align=false - block_size=32768 - index_type=kBinarySearch - filter_policy=rocksdb.BuiltinBloomFilter - metadata_block_size=4096 - no_block_cache=false - whole_key_filtering=false - index_shortening=kShortenSeparators - block_size_deviation=10 - data_block_index_type=kDataBlockBinarySearch - data_block_hash_table_util_ratio=0.750000 - cache_index_and_filter_blocks=false - block_restart_interval=16 - pin_l0_filter_and_index_blocks_in_cache=false - hash_index_allow_collision=true - cache_index_and_filter_blocks_with_high_priority=false - flush_block_policy_factory=FlushBlockBySizePolicyFactory - diff --git a/testdata/singledel_manual_compaction b/testdata/singledel_manual_compaction deleted file mode 100644 index 6b55c9e749e..00000000000 --- a/testdata/singledel_manual_compaction +++ /dev/null @@ -1,77 +0,0 @@ -# This is not actually a manual compaction test, and simply uses manual -# compaction to demonstrate single delete semantics. Specifically, it -# demonstrates that the behavior can be non-deterministic if not used -# correctly. - -# Define a sequence of SET=>SET=>DEL=>SET=>SINGLEDEL. -define target-file-sizes=(1, 1, 1, 1, 1) -L1 - a.SINGLEDEL.10: -L2 - a.SET.9:v3 -L3 - a.DEL.8: -L4 - a.SET.7:v2 -L5 - a.SET.6:v1 ----- -1: - 000004:[a#10,SINGLEDEL-a#10,SINGLEDEL] -2: - 000005:[a#9,SET-a#9,SET] -3: - 000006:[a#8,DEL-a#8,DEL] -4: - 000007:[a#7,SET-a#7,SET] -5: - 000008:[a#6,SET-a#6,SET] - -# No data. -iter -first ----- -. - -# Compact away the DEL. -compact a-b L2 ----- -1: - 000004:[a#10,SINGLEDEL-a#10,SINGLEDEL] -3: - 000009:[a#9,SET-a#9,SET] -4: - 000007:[a#7,SET-a#7,SET] -5: - 000008:[a#6,SET-a#6,SET] - -# No data. -iter -first ----- -. - -# Do two compactions to compact away the SINGLEDEL and 1 SET. -compact a-b L1 ----- -2: - 000010:[a#10,SINGLEDEL-a#10,SINGLEDEL] -3: - 000009:[a#9,SET-a#9,SET] -4: - 000007:[a#7,SET-a#7,SET] -5: - 000008:[a#6,SET-a#6,SET] - -compact a-b L2 ----- -4: - 000007:[a#7,SET-a#7,SET] -5: - 000008:[a#6,SET-a#6,SET] - -# Deleted data reappears. -iter -first ----- -a: (v2, .) diff --git a/tool/db.go b/tool/db.go index dd78daffadb..79b3a767d2d 100644 --- a/tool/db.go +++ b/tool/db.go @@ -531,7 +531,6 @@ func (d *dbT) runProperties(cmd *cobra.Command, args []string) { v, err := bve.Apply( nil /* version */, cmp.Compare, d.fmtKey.fn, d.opts.FlushSplitBytes, d.opts.Experimental.ReadCompactionRate, nil, /* zombies */ - manifest.AllowSplitUserKeys, ) if err != nil { return err diff --git a/tool/manifest.go b/tool/manifest.go index 3cd9fa8ceff..192f929f109 100644 --- a/tool/manifest.go +++ b/tool/manifest.go @@ -244,7 +244,7 @@ func (m *manifestT) runDump(cmd *cobra.Command, args []string) { v, err := bve.Apply( nil /* version */, comparer.Compare, m.fmtKey.fn, 0, m.opts.Experimental.ReadCompactionRate, - nil /* zombies */, manifest.AllowSplitUserKeys, + nil, /* zombies */ ) if err != nil { fmt.Fprintf(stdout, "%s\n", err) @@ -552,7 +552,7 @@ func (m *manifestT) runCheck(cmd *cobra.Command, args []string) { } // TODO(sbhola): add option to Apply that reports all errors instead of // one error. - newv, err := bve.Apply(v, cmp.Compare, m.fmtKey.fn, 0, m.opts.Experimental.ReadCompactionRate, nil /* zombies */, manifest.AllowSplitUserKeys) + newv, err := bve.Apply(v, cmp.Compare, m.fmtKey.fn, 0, m.opts.Experimental.ReadCompactionRate, nil /* zombies */) if err != nil { fmt.Fprintf(stdout, "%s: offset: %d err: %s\n", arg, offset, err) diff --git a/version_set.go b/version_set.go index 35dd0e76fb3..676aec6e3b9 100644 --- a/version_set.go +++ b/version_set.go @@ -5,14 +5,12 @@ package pebble import ( - "bytes" "fmt" "io" "sync" "sync/atomic" "github.com/cockroachdb/errors" - "github.com/cockroachdb/errors/oserror" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/manifest" @@ -116,7 +114,6 @@ type versionSet struct { manifestFile vfs.File manifest *record.Writer - setCurrent func(base.DiskFileNum) error getFormatMajorVersion func() FormatMajorVersion writing bool @@ -129,7 +126,6 @@ func (vs *versionSet) init( dirname string, opts *Options, marker *atomicfs.Marker, - setCurrent func(base.DiskFileNum) error, getFMV func() FormatMajorVersion, mu *sync.Mutex, ) { @@ -148,7 +144,6 @@ func (vs *versionSet) init( vs.backingState.fileBackingSize = 0 vs.nextFileNum = 1 vs.manifestMarker = marker - vs.setCurrent = setCurrent vs.getFormatMajorVersion = getFMV } @@ -158,11 +153,10 @@ func (vs *versionSet) create( dirname string, opts *Options, marker *atomicfs.Marker, - setCurrent func(base.DiskFileNum) error, getFormatMajorVersion func() FormatMajorVersion, mu *sync.Mutex, ) error { - vs.init(dirname, opts, marker, setCurrent, getFormatMajorVersion, mu) + vs.init(dirname, opts, marker, getFormatMajorVersion, mu) newVersion := &version{} vs.append(newVersion) var err error @@ -183,8 +177,8 @@ func (vs *versionSet) create( } } if err == nil { - // NB: setCurrent is responsible for syncing the data directory. - if err = vs.setCurrent(vs.manifestFileNum); err != nil { + // NB: Move() is responsible for syncing the data directory. + if err = vs.manifestMarker.Move(base.MakeFilename(fileTypeManifest, vs.manifestFileNum)); err != nil { vs.opts.Logger.Fatalf("MANIFEST set current failed: %v", err) } } @@ -207,11 +201,10 @@ func (vs *versionSet) load( opts *Options, manifestFileNum base.DiskFileNum, marker *atomicfs.Marker, - setCurrent func(base.DiskFileNum) error, getFormatMajorVersion func() FormatMajorVersion, mu *sync.Mutex, ) error { - vs.init(dirname, opts, marker, setCurrent, getFormatMajorVersion, mu) + vs.init(dirname, opts, marker, getFormatMajorVersion, mu) vs.manifestFileNum = manifestFileNum manifestPath := base.MakeFilepath(opts.FS, dirname, fileTypeManifest, vs.manifestFileNum) @@ -310,7 +303,6 @@ func (vs *versionSet) load( newVersion, err := bve.Apply( nil, vs.cmp, opts.Comparer.FormatKey, opts.FlushSplitBytes, opts.Experimental.ReadCompactionRate, nil, /* zombies */ - getFormatMajorVersion().orderingInvariants(), ) if err != nil { return err @@ -445,8 +437,6 @@ func (vs *versionSet) logAndApply( } currentVersion := vs.currentVersion() - fmv := vs.getFormatMajorVersion() - orderingInvariants := fmv.orderingInvariants() var newVersion *version // Generate a new manifest if we don't currently have one, or forceRotation @@ -524,7 +514,6 @@ func (vs *versionSet) logAndApply( ve, currentVersion, vs.cmp, vs.opts.Comparer.FormatKey, vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate, vs.backingState.fileBackingMap, vs.addFileBacking, vs.removeFileBacking, - orderingInvariants, ) if err != nil { return errors.Wrap(err, "MANIFEST apply failed") @@ -562,8 +551,8 @@ func (vs *versionSet) logAndApply( return errors.Wrap(err, "MANIFEST sync failed") } if newManifestFileNum != 0 { - // NB: setCurrent is responsible for syncing the data directory. - if err := vs.setCurrent(newManifestFileNum); err != nil { + // NB: Move() is responsible for syncing the data directory. + if err := vs.manifestMarker.Move(base.MakeFilename(fileTypeManifest, newManifestFileNum)); err != nil { return errors.Wrap(err, "MANIFEST set current failed") } vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{ @@ -883,69 +872,16 @@ func (vs *versionSet) updateObsoleteTableMetricsLocked() { } } -func setCurrentFunc( - vers FormatMajorVersion, marker *atomicfs.Marker, fs vfs.FS, dirname string, dir vfs.File, -) func(base.DiskFileNum) error { - if vers < formatVersionedManifestMarker { - // Pebble versions before `formatVersionedManifestMarker` used - // the CURRENT file to signal which MANIFEST is current. Ignore - // the filename read during LocateMarker. - return func(manifestFileNum base.DiskFileNum) error { - if err := setCurrentFile(dirname, fs, manifestFileNum); err != nil { - return err - } - if err := dir.Sync(); err != nil { - // This is a panic here, rather than higher in the call - // stack, for parity with the atomicfs.Marker behavior. - // A panic is always necessary because failed Syncs are - // unrecoverable. - panic(errors.Wrap(err, "fatal: MANIFEST dirsync failed")) - } - return nil - } - } - return setCurrentFuncMarker(marker, fs, dirname) -} - -func setCurrentFuncMarker( - marker *atomicfs.Marker, fs vfs.FS, dirname string, -) func(base.DiskFileNum) error { - return func(manifestFileNum base.DiskFileNum) error { - return marker.Move(base.MakeFilename(fileTypeManifest, manifestFileNum)) - } -} - func findCurrentManifest( - vers FormatMajorVersion, fs vfs.FS, dirname string, + fs vfs.FS, dirname string, ) (marker *atomicfs.Marker, manifestNum base.DiskFileNum, exists bool, err error) { - // NB: We always locate the manifest marker, even if we might not - // actually use it (because we're opening the database at an earlier - // format major version that uses the CURRENT file). Locating a - // marker should succeed even if the marker has never been placed. + // Locating a marker should succeed even if the marker has never been placed. var filename string marker, filename, err = atomicfs.LocateMarker(fs, dirname, manifestMarkerName) if err != nil { return nil, base.FileNum(0).DiskFileNum(), false, err } - if vers < formatVersionedManifestMarker { - // Pebble versions before `formatVersionedManifestMarker` used - // the CURRENT file to signal which MANIFEST is current. Ignore - // the filename read during LocateMarker. - - manifestNum, err = readCurrentFile(fs, dirname) - if oserror.IsNotExist(err) { - return marker, base.FileNum(0).DiskFileNum(), false, nil - } else if err != nil { - return marker, base.FileNum(0).DiskFileNum(), false, err - } - return marker, manifestNum, true, nil - } - - // The current format major version is >= - // formatVersionedManifestMarker indicating that the - // atomicfs.Marker is the source of truth on the current manifest. - if filename == "" { // The marker hasn't been set yet. This database doesn't exist. return marker, base.FileNum(0).DiskFileNum(), false, nil @@ -959,41 +895,6 @@ func findCurrentManifest( return marker, manifestNum, true, nil } -func readCurrentFile(fs vfs.FS, dirname string) (base.DiskFileNum, error) { - // Read the CURRENT file to find the current manifest file. - current, err := fs.Open(base.MakeFilepath(fs, dirname, fileTypeCurrent, base.FileNum(0).DiskFileNum())) - if err != nil { - return base.FileNum(0).DiskFileNum(), errors.Wrapf(err, "pebble: could not open CURRENT file for DB %q", dirname) - } - defer current.Close() - stat, err := current.Stat() - if err != nil { - return base.FileNum(0).DiskFileNum(), err - } - n := stat.Size() - if n == 0 { - return base.FileNum(0).DiskFileNum(), errors.Errorf("pebble: CURRENT file for DB %q is empty", dirname) - } - if n > 4096 { - return base.FileNum(0).DiskFileNum(), errors.Errorf("pebble: CURRENT file for DB %q is too large", dirname) - } - b := make([]byte, n) - _, err = current.ReadAt(b, 0) - if err != nil { - return base.FileNum(0).DiskFileNum(), err - } - if b[n-1] != '\n' { - return base.FileNum(0).DiskFileNum(), base.CorruptionErrorf("pebble: CURRENT file for DB %q is malformed", dirname) - } - b = bytes.TrimSpace(b) - - _, manifestFileNum, ok := base.ParseFilename(fs, string(b)) - if !ok { - return base.FileNum(0).DiskFileNum(), base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(b)) - } - return manifestFileNum, nil -} - func newFileMetrics(newFiles []manifest.NewFileEntry) map[int]*LevelMetrics { m := map[int]*LevelMetrics{} for _, nf := range newFiles {