diff --git a/db_test.go b/db_test.go index e917778385..c444a4982a 100644 --- a/db_test.go +++ b/db_test.go @@ -1432,7 +1432,7 @@ func TestTracing(t *testing.T) { _, closer, err := d.Get([]byte("hello")) require.NoError(t, err) closer.Close() - readerInitTraceString := "reading 37 bytes took 5ms\nreading 491 bytes took 5ms\n" + readerInitTraceString := "reading 37 bytes took 5ms\nreading 419 bytes took 5ms\n" iterTraceString := "reading 27 bytes took 5ms\nreading 29 bytes took 5ms\n" require.Equal(t, readerInitTraceString+iterTraceString, tracer.buf.String()) diff --git a/replay/testdata/corpus/simple b/replay/testdata/corpus/simple index 1b758b5820..28d6a51dd8 100644 --- a/replay/testdata/corpus/simple +++ b/replay/testdata/corpus/simple @@ -82,4 +82,4 @@ simple/MANIFEST-000001: simple/MANIFEST-000008: size: 122 simple/000007.sst: - size: 686 + size: 614 diff --git a/replay/testdata/replay b/replay/testdata/replay index b7df7a4690..17c837be43 100644 --- a/replay/testdata/replay +++ b/replay/testdata/replay @@ -6,9 +6,9 @@ tree / build/ 89 000004.log - 658 000005.sst + 586 000005.sst 49 000006.log - 686 000007.sst + 614 000007.sst 0 LOCK 98 MANIFEST-000001 122 MANIFEST-000008 @@ -16,12 +16,12 @@ tree 0 marker.format-version.000001.013 0 marker.manifest.000002.MANIFEST-000008 simple/ - 686 000007.sst + 614 000007.sst 98 MANIFEST-000001 122 MANIFEST-000008 checkpoint/ 25 000004.log - 658 000005.sst + 586 000005.sst 98 MANIFEST-000001 1211 OPTIONS-000003 0 marker.format-version.000001.013 diff --git a/replay/testdata/replay_paced b/replay/testdata/replay_paced index 019ca0a7cc..c3c667575e 100644 --- a/replay/testdata/replay_paced +++ b/replay/testdata/replay_paced @@ -5,12 +5,12 @@ tree ---- / build/ - 936 000005.sst - 632 000007.sst + 864 000005.sst + 560 000007.sst 89 000009.log - 632 000010.sst + 560 000010.sst 200 000012.log - 686 000013.sst + 614 000013.sst 0 LOCK 122 MANIFEST-000008 205 MANIFEST-000011 @@ -18,13 +18,13 @@ tree 0 marker.format-version.000001.013 0 marker.manifest.000003.MANIFEST-000011 high_read_amp/ - 686 000013.sst + 614 000013.sst 205 MANIFEST-000011 checkpoint/ - 936 000005.sst - 632 000007.sst + 864 000005.sst + 560 000007.sst 39 000009.log - 632 000010.sst + 560 000010.sst 157 MANIFEST-000011 1211 OPTIONS-000003 0 marker.format-version.000001.013 diff --git a/sstable/properties.go b/sstable/properties.go index 2a3a92fb95..e9a06b9b87 100644 --- a/sstable/properties.go +++ b/sstable/properties.go @@ -163,11 +163,6 @@ type Properties struct { NumValueBlocks uint64 `prop:"pebble.num.value-blocks"` // The number of values stored in value blocks. Only serialized if > 0. NumValuesInValueBlocks uint64 `prop:"pebble.num.values.in.value-blocks"` - // The name of the prefix extractor used in this table. Empty if no prefix - // extractor is used. - PrefixExtractorName string `prop:"rocksdb.prefix.extractor.name"` - // If filtering is enabled, was the filter created on the key prefix. - PrefixFiltering bool `prop:"rocksdb.block.based.table.prefix.filtering"` // A comma separated list of names of the property collectors used in this // table. PropertyCollectorNames string `prop:"rocksdb.property.collectors"` @@ -187,8 +182,6 @@ type Properties struct { TopLevelIndexSize uint64 `prop:"rocksdb.top-level.index.size"` // User collected properties. UserProperties map[string]string - // If filtering is enabled, was the filter created on the whole key. - WholeKeyFiltering bool `prop:"rocksdb.block.based.table.whole.key.filtering"` // Loaded set indicating which fields have been loaded from disk. Indexed by // the field's byte offset within the struct @@ -402,10 +395,6 @@ func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) { if p.NumValuesInValueBlocks > 0 { p.saveUvarint(m, unsafe.Offsetof(p.NumValuesInValueBlocks), p.NumValuesInValueBlocks) } - if p.PrefixExtractorName != "" { - p.saveString(m, unsafe.Offsetof(p.PrefixExtractorName), p.PrefixExtractorName) - } - p.saveBool(m, unsafe.Offsetof(p.PrefixFiltering), p.PrefixFiltering) if p.PropertyCollectorNames != "" { p.saveString(m, unsafe.Offsetof(p.PropertyCollectorNames), p.PropertyCollectorNames) } @@ -419,7 +408,6 @@ func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) { if p.ValueBlocksSize > 0 { p.saveUvarint(m, unsafe.Offsetof(p.ValueBlocksSize), p.ValueBlocksSize) } - p.saveBool(m, unsafe.Offsetof(p.WholeKeyFiltering), p.WholeKeyFiltering) if tblFormat < TableFormatPebblev1 { m["rocksdb.column.family.id"] = binary.AppendUvarint([]byte(nil), math.MaxInt32) diff --git a/sstable/properties_test.go b/sstable/properties_test.go index 7890125003..7b9cffd90d 100644 --- a/sstable/properties_test.go +++ b/sstable/properties_test.go @@ -35,9 +35,7 @@ func TestPropertiesLoad(t *testing.T) { IndexSize: 325, MergerName: "nullptr", NumDataBlocks: 14, - PrefixExtractorName: "nullptr", PropertyCollectorNames: "[]", - WholeKeyFiltering: false, } { @@ -85,11 +83,8 @@ var testProps = Properties{ NumRangeKeyUnsets: 21, NumValueBlocks: 22, NumValuesInValueBlocks: 23, - PrefixExtractorName: "prefix extractor name", - PrefixFiltering: true, PropertyCollectorNames: "prefix collector names", TopLevelIndexSize: 27, - WholeKeyFiltering: true, UserProperties: map[string]string{ "user-prop-a": "1", "user-prop-b": "2", diff --git a/sstable/testdata/h.no-compression.sst b/sstable/testdata/h.no-compression.sst index 1bf742e8ea..771d1a3830 100644 Binary files a/sstable/testdata/h.no-compression.sst and b/sstable/testdata/h.no-compression.sst differ diff --git a/sstable/testdata/h.no-compression.two_level_index.sst b/sstable/testdata/h.no-compression.two_level_index.sst index 5e9a68b253..8661224ae6 100644 Binary files a/sstable/testdata/h.no-compression.two_level_index.sst and b/sstable/testdata/h.no-compression.two_level_index.sst differ diff --git a/sstable/testdata/h.sst b/sstable/testdata/h.sst index b7ff583532..507002a50a 100644 Binary files a/sstable/testdata/h.sst and b/sstable/testdata/h.sst differ diff --git a/sstable/testdata/h.table-bloom.no-compression.prefix_extractor.no_whole_key_filter.sst b/sstable/testdata/h.table-bloom.no-compression.prefix_extractor.no_whole_key_filter.sst index 000cd2e089..b359032af3 100644 Binary files a/sstable/testdata/h.table-bloom.no-compression.prefix_extractor.no_whole_key_filter.sst and b/sstable/testdata/h.table-bloom.no-compression.prefix_extractor.no_whole_key_filter.sst differ diff --git a/sstable/testdata/h.table-bloom.no-compression.sst b/sstable/testdata/h.table-bloom.no-compression.sst index b2e2681985..b359032af3 100644 Binary files a/sstable/testdata/h.table-bloom.no-compression.sst and b/sstable/testdata/h.table-bloom.no-compression.sst differ diff --git a/sstable/testdata/h.table-bloom.sst b/sstable/testdata/h.table-bloom.sst index 3b512372a8..a2a4f67279 100644 Binary files a/sstable/testdata/h.table-bloom.sst and b/sstable/testdata/h.table-bloom.sst differ diff --git a/sstable/testdata/h.zstd-compression.sst b/sstable/testdata/h.zstd-compression.sst index d0b6e4b927..57205b5fde 100644 Binary files a/sstable/testdata/h.zstd-compression.sst and b/sstable/testdata/h.zstd-compression.sst differ diff --git a/sstable/testdata/rewriter b/sstable/testdata/rewriter index b04882d47d..ab41ea8a29 100644 --- a/sstable/testdata/rewriter +++ b/sstable/testdata/rewriter @@ -48,10 +48,10 @@ layout 191 index (22) 218 index (22) 245 top-index (48) - 298 properties (607) - 910 meta-index (79) - 994 footer (53) - 1047 EOF + 298 properties (518) + 821 meta-index (79) + 905 footer (53) + 958 EOF scan ---- @@ -83,10 +83,10 @@ layout 191 index (22) 218 index (22) 245 top-index (48) - 298 properties (607) - 910 meta-index (79) - 994 footer (53) - 1047 EOF + 298 properties (518) + 821 meta-index (79) + 905 footer (53) + 958 EOF scan ---- @@ -118,10 +118,10 @@ layout 191 index (22) 218 index (22) 245 top-index (48) - 298 properties (607) - 910 meta-index (79) - 994 footer (53) - 1047 EOF + 298 properties (518) + 821 meta-index (79) + 905 footer (53) + 958 EOF scan ---- @@ -153,10 +153,10 @@ layout 191 index (22) 218 index (22) 245 top-index (48) - 298 properties (607) - 910 meta-index (79) - 994 footer (53) - 1047 EOF + 298 properties (518) + 821 meta-index (79) + 905 footer (53) + 958 EOF scan ---- @@ -189,10 +189,10 @@ layout 191 index (22) 218 index (22) 245 top-index (48) - 298 properties (607) - 910 meta-index (79) - 994 footer (53) - 1047 EOF + 298 properties (518) + 821 meta-index (79) + 905 footer (53) + 958 EOF scan ---- diff --git a/sstable/testdata/rewriter_v3 b/sstable/testdata/rewriter_v3 index 6ec3dc558b..38ca0cd940 100644 --- a/sstable/testdata/rewriter_v3 +++ b/sstable/testdata/rewriter_v3 @@ -48,10 +48,10 @@ layout 194 index (22) 221 index (22) 248 top-index (48) - 301 properties (607) - 913 meta-index (79) - 997 footer (53) - 1050 EOF + 301 properties (518) + 824 meta-index (79) + 908 footer (53) + 961 EOF scan ---- @@ -83,10 +83,10 @@ layout 194 index (22) 221 index (22) 248 top-index (48) - 301 properties (607) - 913 meta-index (79) - 997 footer (53) - 1050 EOF + 301 properties (518) + 824 meta-index (79) + 908 footer (53) + 961 EOF scan ---- @@ -118,10 +118,10 @@ layout 194 index (22) 221 index (22) 248 top-index (48) - 301 properties (607) - 913 meta-index (79) - 997 footer (53) - 1050 EOF + 301 properties (518) + 824 meta-index (79) + 908 footer (53) + 961 EOF scan ---- @@ -153,10 +153,10 @@ layout 194 index (22) 221 index (22) 248 top-index (48) - 301 properties (607) - 913 meta-index (79) - 997 footer (53) - 1050 EOF + 301 properties (518) + 824 meta-index (79) + 908 footer (53) + 961 EOF scan ---- @@ -189,10 +189,10 @@ layout 194 index (22) 221 index (22) 248 top-index (48) - 301 properties (607) - 913 meta-index (79) - 997 footer (53) - 1050 EOF + 301 properties (518) + 824 meta-index (79) + 908 footer (53) + 961 EOF scan ---- diff --git a/sstable/testdata/virtual_reader b/sstable/testdata/virtual_reader index 5977c6ecd0..9b3a8e1eb0 100644 --- a/sstable/testdata/virtual_reader +++ b/sstable/testdata/virtual_reader @@ -159,7 +159,7 @@ virtualize dd.SET.5-ddd.SET.6 ---- bounds: [dd#5,1-ddd#6,1] filenum: 000004 -props: NumEntries: 2, RawKeySize: 10, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 0 +props: NumEntries: 2, RawKeySize: 11, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 0 # Check lower bound enforcement during SeekPrefixGE. iter @@ -190,7 +190,7 @@ virtualize c.SET.3-f.SET.6 ---- bounds: [c#3,1-f#6,1] filenum: 000005 -props: NumEntries: 2, RawKeySize: 10, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 0 +props: NumEntries: 2, RawKeySize: 11, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 0 # Just test a basic iterator once virtual sstable bounds have been set. iter @@ -290,7 +290,7 @@ virtualize c.SET.3-f.SET.1:ff ---- bounds: [c#3,1-f#0,1] filenum: 000006 -props: NumEntries: 2, RawKeySize: 12, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 3 +props: NumEntries: 2, RawKeySize: 13, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 3 iter set-bounds lower=d upper=e @@ -343,7 +343,7 @@ virtualize f.SET.6-h.SET.9 ---- bounds: [f#6,1-h#9,1] filenum: 000007 -props: NumEntries: 2, RawKeySize: 12, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 3 +props: NumEntries: 2, RawKeySize: 13, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 3 iter seek-lt z @@ -713,7 +713,7 @@ virtualize c@7.SET.3-f@4.SET.8 suffix=@8 ---- bounds: [c@7#3,1-f@4#8,1] filenum: 000015 -props: NumEntries: 2, RawKeySize: 13, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 0 +props: NumEntries: 2, RawKeySize: 15, RawValueSize: 2, RawPointTombstoneKeySize: 0, RawPointTombstoneValueSize: 0, NumSizedDeletions: 0, NumDeletions: 0, NumRangeDeletions: 0, NumRangeKeyDels: 0, NumRangeKeySets: 0, ValueBlocksSize: 0 # Just test a basic iterator once virtual sstable bounds have been set. iter diff --git a/sstable/testdata/writer b/sstable/testdata/writer index 085c7978f2..32182a4650 100644 --- a/sstable/testdata/writer +++ b/sstable/testdata/writer @@ -316,10 +316,10 @@ layout 105 index (22) 132 index (22) 159 top-index (50) - 214 properties (557) - 776 meta-index (33) - 814 footer (53) - 867 EOF + 214 properties (485) + 704 meta-index (33) + 742 footer (53) + 795 EOF scan ---- @@ -344,10 +344,10 @@ layout 26 data (21) 52 data (21) 78 index (47) - 130 properties (655) - 790 meta-index (33) - 828 leveldb-footer (48) - 876 EOF + 130 properties (583) + 718 meta-index (33) + 756 leveldb-footer (48) + 804 EOF # Range keys, if present, are shown in the layout. @@ -364,7 +364,7 @@ layout 0 data (8) 13 index (21) 39 range-key (82) - 126 properties (605) - 736 meta-index (57) - 798 footer (53) - 851 EOF + 126 properties (533) + 664 meta-index (57) + 726 footer (53) + 779 EOF diff --git a/sstable/testdata/writer_v3 b/sstable/testdata/writer_v3 index 5ff7a4a046..e509d60491 100644 --- a/sstable/testdata/writer_v3 +++ b/sstable/testdata/writer_v3 @@ -289,10 +289,10 @@ layout 108 index (22) 135 index (22) 162 top-index (51) - 218 properties (557) - 780 meta-index (33) - 818 footer (53) - 871 EOF + 218 properties (485) + 708 meta-index (33) + 746 footer (53) + 799 EOF scan ---- @@ -317,10 +317,10 @@ layout 26 data (21) 52 data (21) 78 index (47) - 130 properties (655) - 790 meta-index (33) - 828 leveldb-footer (48) - 876 EOF + 130 properties (583) + 718 meta-index (33) + 756 leveldb-footer (48) + 804 EOF # Range keys, if present, are shown in the layout. @@ -337,7 +337,7 @@ layout 0 data (8) 13 index (21) 39 range-key (82) - 126 properties (605) - 736 meta-index (57) - 798 footer (53) - 851 EOF + 126 properties (533) + 664 meta-index (57) + 726 footer (53) + 779 EOF diff --git a/sstable/testdata/writer_value_blocks b/sstable/testdata/writer_value_blocks index 5e09d6ac34..e458abcbc3 100644 --- a/sstable/testdata/writer_value_blocks +++ b/sstable/testdata/writer_value_blocks @@ -193,49 +193,46 @@ layout 357 value-block (11) 373 value-block (15) 393 value-index (8) - 406 properties (653) + 406 properties (581) 406 obsolete-key (16) [restart] 422 pebble.num.value-blocks (27) 449 pebble.num.values.in.value-blocks (21) 470 pebble.value-blocks.size (21) 491 rocksdb.block.based.table.index.type (43) - 534 rocksdb.block.based.table.prefix.filtering (20) - 554 rocksdb.block.based.table.whole.key.filtering (23) - 577 rocksdb.comparator (37) - 614 rocksdb.compression (16) - 630 rocksdb.compression_options (106) - 736 rocksdb.data.size (14) - 750 rocksdb.deleted.keys (15) - 765 rocksdb.external_sst_file.version (32) - 797 rocksdb.filter.size (15) - 812 rocksdb.index.partitions (20) - 832 rocksdb.index.size (9) - 841 rocksdb.merge.operands (18) - 859 rocksdb.merge.operator (24) - 883 rocksdb.num.data.blocks (19) - 902 rocksdb.num.entries (11) - 913 rocksdb.num.range-deletions (19) - 932 rocksdb.prefix.extractor.name (31) - 963 rocksdb.property.collectors (34) - 997 rocksdb.raw.key.size (16) - 1013 rocksdb.raw.value.size (14) - 1027 rocksdb.top-level.index.size (24) - 1051 [restart 406] - 1059 [trailer compression=none checksum=0x98bc4d71] - 1064 meta-index (64) - 1064 pebble.value_index block:393/8 value-blocks-index-lengths: 1(num), 2(offset), 1(length) [restart] - 1091 rocksdb.properties block:406/653 [restart] - 1116 [restart 1064] - 1120 [restart 1091] - 1128 [trailer compression=none checksum=0xdf6cf118] - 1133 footer (53) - 1133 checksum type: crc32c - 1134 meta: offset=1064, length=64 - 1137 index: offset=267, length=85 - 1140 [padding] - 1174 version: 4 - 1178 magic number: 0xf09faab3f09faab3 - 1186 EOF + 534 rocksdb.comparator (37) + 571 rocksdb.compression (16) + 587 rocksdb.compression_options (106) + 693 rocksdb.data.size (14) + 707 rocksdb.deleted.keys (15) + 722 rocksdb.external_sst_file.version (32) + 754 rocksdb.filter.size (15) + 769 rocksdb.index.partitions (20) + 789 rocksdb.index.size (9) + 798 rocksdb.merge.operands (18) + 816 rocksdb.merge.operator (24) + 840 rocksdb.num.data.blocks (19) + 859 rocksdb.num.entries (11) + 870 rocksdb.num.range-deletions (19) + 889 rocksdb.property.collectors (36) + 925 rocksdb.raw.key.size (16) + 941 rocksdb.raw.value.size (14) + 955 rocksdb.top-level.index.size (24) + 979 [restart 406] + 987 [trailer compression=none checksum=0x7984bf0d] + 992 meta-index (64) + 992 pebble.value_index block:393/8 value-blocks-index-lengths: 1(num), 2(offset), 1(length) [restart] + 1019 rocksdb.properties block:406/581 [restart] + 1044 [restart 992] + 1048 [restart 1019] + 1056 [trailer compression=none checksum=0x4e86cfff] + 1061 footer (53) + 1061 checksum type: crc32c + 1062 meta: offset=992, length=64 + 1065 index: offset=267, length=85 + 1068 [padding] + 1102 version: 4 + 1106 magic number: 0xf09faab3f09faab3 + 1114 EOF # Require that [c,e) must be in-place. build in-place-bound=(c,e) @@ -315,40 +312,37 @@ layout 71 block:0/66 [restart] 85 [restart 71] 93 [trailer compression=none checksum=0xf80f5bcf] - 98 properties (583) + 98 properties (511) 98 obsolete-key (16) [restart] 114 pebble.raw.point-tombstone.key.size (39) 153 rocksdb.block.based.table.index.type (43) - 196 rocksdb.block.based.table.prefix.filtering (20) - 216 rocksdb.block.based.table.whole.key.filtering (23) - 239 rocksdb.comparator (37) - 276 rocksdb.compression (16) - 292 rocksdb.compression_options (106) - 398 rocksdb.data.size (13) - 411 rocksdb.deleted.keys (15) - 426 rocksdb.external_sst_file.version (32) - 458 rocksdb.filter.size (15) - 473 rocksdb.index.size (14) - 487 rocksdb.merge.operands (18) - 505 rocksdb.merge.operator (24) - 529 rocksdb.num.data.blocks (19) - 548 rocksdb.num.entries (11) - 559 rocksdb.num.range-deletions (19) - 578 rocksdb.prefix.extractor.name (31) - 609 rocksdb.property.collectors (34) - 643 rocksdb.raw.key.size (16) - 659 rocksdb.raw.value.size (14) - 673 [restart 98] - 681 [trailer compression=none checksum=0xeb626b4c] - 686 meta-index (32) - 686 rocksdb.properties block:98/583 [restart] - 710 [restart 686] - 718 [trailer compression=none checksum=0x55c264c2] - 723 footer (53) - 723 checksum type: crc32c - 724 meta: offset=686, length=32 - 727 index: offset=71, length=22 - 729 [padding] - 764 version: 4 - 768 magic number: 0xf09faab3f09faab3 - 776 EOF + 196 rocksdb.comparator (37) + 233 rocksdb.compression (16) + 249 rocksdb.compression_options (106) + 355 rocksdb.data.size (13) + 368 rocksdb.deleted.keys (15) + 383 rocksdb.external_sst_file.version (32) + 415 rocksdb.filter.size (15) + 430 rocksdb.index.size (14) + 444 rocksdb.merge.operands (18) + 462 rocksdb.merge.operator (24) + 486 rocksdb.num.data.blocks (19) + 505 rocksdb.num.entries (11) + 516 rocksdb.num.range-deletions (19) + 535 rocksdb.property.collectors (36) + 571 rocksdb.raw.key.size (16) + 587 rocksdb.raw.value.size (14) + 601 [restart 98] + 609 [trailer compression=none checksum=0x82ffde38] + 614 meta-index (32) + 614 rocksdb.properties block:98/511 [restart] + 638 [restart 614] + 646 [trailer compression=none checksum=0x8b6604da] + 651 footer (53) + 651 checksum type: crc32c + 652 meta: offset=614, length=32 + 655 index: offset=71, length=22 + 657 [padding] + 692 version: 4 + 696 magic number: 0xf09faab3f09faab3 + 704 EOF diff --git a/sstable/writer.go b/sstable/writer.go index 80a578480f..4968cfb5b1 100644 --- a/sstable/writer.go +++ b/sstable/writer.go @@ -2251,17 +2251,10 @@ func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...Write } } - w.props.PrefixExtractorName = "nullptr" if o.FilterPolicy != nil { switch o.FilterType { case TableFilter: w.filter = newTableFilterWriter(o.FilterPolicy) - if w.split != nil { - w.props.PrefixExtractorName = o.Comparer.Name - w.props.PrefixFiltering = true - } else { - w.props.WholeKeyFiltering = true - } default: panic(fmt.Sprintf("unknown filter type: %v", o.FilterType)) } diff --git a/testdata/checkpoint b/testdata/checkpoint index 4ee3f11d27..267bc0964d 100644 --- a/testdata/checkpoint +++ b/testdata/checkpoint @@ -210,17 +210,17 @@ close: db/000009.sst sync: db sync: db/MANIFEST-000001 open: db/000005.sst -read-at(630, 53): db/000005.sst -read-at(593, 37): db/000005.sst -read-at(74, 519): db/000005.sst +read-at(558, 53): db/000005.sst +read-at(521, 37): db/000005.sst +read-at(74, 447): db/000005.sst open: db/000009.sst -read-at(625, 53): db/000009.sst -read-at(588, 37): db/000009.sst -read-at(69, 519): db/000009.sst +read-at(553, 53): db/000009.sst +read-at(516, 37): db/000009.sst +read-at(69, 447): db/000009.sst open: db/000007.sst -read-at(630, 53): db/000007.sst -read-at(593, 37): db/000007.sst -read-at(74, 519): db/000007.sst +read-at(558, 53): db/000007.sst +read-at(521, 37): db/000007.sst +read-at(74, 447): db/000007.sst read-at(47, 27): db/000005.sst open: db/000005.sst read-at(0, 47): db/000005.sst @@ -289,15 +289,15 @@ close: checkpoints/checkpoint1/000006.log scan checkpoints/checkpoint1 ---- open: checkpoints/checkpoint1/000007.sst -read-at(630, 53): checkpoints/checkpoint1/000007.sst -read-at(593, 37): checkpoints/checkpoint1/000007.sst -read-at(74, 519): checkpoints/checkpoint1/000007.sst +read-at(558, 53): checkpoints/checkpoint1/000007.sst +read-at(521, 37): checkpoints/checkpoint1/000007.sst +read-at(74, 447): checkpoints/checkpoint1/000007.sst read-at(47, 27): checkpoints/checkpoint1/000007.sst read-at(0, 47): checkpoints/checkpoint1/000007.sst open: checkpoints/checkpoint1/000005.sst -read-at(630, 53): checkpoints/checkpoint1/000005.sst -read-at(593, 37): checkpoints/checkpoint1/000005.sst -read-at(74, 519): checkpoints/checkpoint1/000005.sst +read-at(558, 53): checkpoints/checkpoint1/000005.sst +read-at(521, 37): checkpoints/checkpoint1/000005.sst +read-at(74, 447): checkpoints/checkpoint1/000005.sst read-at(47, 27): checkpoints/checkpoint1/000005.sst read-at(0, 47): checkpoints/checkpoint1/000005.sst a 1 @@ -312,9 +312,9 @@ g 10 scan db ---- open: db/000010.sst -read-at(657, 53): db/000010.sst -read-at(620, 37): db/000010.sst -read-at(101, 519): db/000010.sst +read-at(585, 53): db/000010.sst +read-at(548, 37): db/000010.sst +read-at(101, 447): db/000010.sst read-at(74, 27): db/000010.sst read-at(0, 74): db/000010.sst a 1 @@ -355,9 +355,9 @@ close: checkpoints/checkpoint2/000006.log scan checkpoints/checkpoint2 ---- open: checkpoints/checkpoint2/000007.sst -read-at(630, 53): checkpoints/checkpoint2/000007.sst -read-at(593, 37): checkpoints/checkpoint2/000007.sst -read-at(74, 519): checkpoints/checkpoint2/000007.sst +read-at(558, 53): checkpoints/checkpoint2/000007.sst +read-at(521, 37): checkpoints/checkpoint2/000007.sst +read-at(74, 447): checkpoints/checkpoint2/000007.sst read-at(47, 27): checkpoints/checkpoint2/000007.sst read-at(0, 47): checkpoints/checkpoint2/000007.sst b 5 @@ -396,15 +396,15 @@ close: checkpoints/checkpoint3/000006.log scan checkpoints/checkpoint3 ---- open: checkpoints/checkpoint3/000007.sst -read-at(630, 53): checkpoints/checkpoint3/000007.sst -read-at(593, 37): checkpoints/checkpoint3/000007.sst -read-at(74, 519): checkpoints/checkpoint3/000007.sst +read-at(558, 53): checkpoints/checkpoint3/000007.sst +read-at(521, 37): checkpoints/checkpoint3/000007.sst +read-at(74, 447): checkpoints/checkpoint3/000007.sst read-at(47, 27): checkpoints/checkpoint3/000007.sst read-at(0, 47): checkpoints/checkpoint3/000007.sst open: checkpoints/checkpoint3/000005.sst -read-at(630, 53): checkpoints/checkpoint3/000005.sst -read-at(593, 37): checkpoints/checkpoint3/000005.sst -read-at(74, 519): checkpoints/checkpoint3/000005.sst +read-at(558, 53): checkpoints/checkpoint3/000005.sst +read-at(521, 37): checkpoints/checkpoint3/000005.sst +read-at(74, 447): checkpoints/checkpoint3/000005.sst read-at(47, 27): checkpoints/checkpoint3/000005.sst read-at(0, 47): checkpoints/checkpoint3/000005.sst a 1 @@ -473,8 +473,8 @@ h 11 i i k k open: db/000014.sst -read-at(613, 53): db/000014.sst -read-at(576, 37): db/000014.sst +read-at(541, 53): db/000014.sst +read-at(504, 37): db/000014.sst z z . @@ -533,9 +533,9 @@ close: checkpoints/checkpoint4/000008.log scan checkpoints/checkpoint4 ---- open: checkpoints/checkpoint4/000010.sst -read-at(657, 53): checkpoints/checkpoint4/000010.sst -read-at(620, 37): checkpoints/checkpoint4/000010.sst -read-at(101, 519): checkpoints/checkpoint4/000010.sst +read-at(585, 53): checkpoints/checkpoint4/000010.sst +read-at(548, 37): checkpoints/checkpoint4/000010.sst +read-at(101, 447): checkpoints/checkpoint4/000010.sst read-at(74, 27): checkpoints/checkpoint4/000010.sst read-at(0, 74): checkpoints/checkpoint4/000010.sst a 1 @@ -545,18 +545,18 @@ e 8 f 9 g 10 open: checkpoints/checkpoint4/000011.sst -read-at(630, 53): checkpoints/checkpoint4/000011.sst -read-at(593, 37): checkpoints/checkpoint4/000011.sst -read-at(70, 523): checkpoints/checkpoint4/000011.sst +read-at(558, 53): checkpoints/checkpoint4/000011.sst +read-at(521, 37): checkpoints/checkpoint4/000011.sst +read-at(70, 451): checkpoints/checkpoint4/000011.sst read-at(43, 27): checkpoints/checkpoint4/000011.sst read-at(0, 43): checkpoints/checkpoint4/000011.sst h 11 i i k k open: checkpoints/checkpoint4/000014.sst -read-at(613, 53): checkpoints/checkpoint4/000014.sst -read-at(576, 37): checkpoints/checkpoint4/000014.sst -read-at(53, 523): checkpoints/checkpoint4/000014.sst +read-at(541, 53): checkpoints/checkpoint4/000014.sst +read-at(504, 37): checkpoints/checkpoint4/000014.sst +read-at(53, 451): checkpoints/checkpoint4/000014.sst read-at(26, 27): checkpoints/checkpoint4/000014.sst read-at(0, 26): checkpoints/checkpoint4/000014.sst z z diff --git a/testdata/cleaner b/testdata/cleaner index 38f3c22d5a..4470cb388f 100644 --- a/testdata/cleaner +++ b/testdata/cleaner @@ -76,13 +76,13 @@ sync: db/MANIFEST-000001 mkdir-all: db_wal/archive 0755 rename: db_wal/000004.log -> db_wal/archive/000004.log open: db/000005.sst -read-at(607, 53): db/000005.sst -read-at(570, 37): db/000005.sst -read-at(79, 491): db/000005.sst +read-at(535, 53): db/000005.sst +read-at(498, 37): db/000005.sst +read-at(79, 419): db/000005.sst open: db/000007.sst -read-at(581, 53): db/000007.sst -read-at(544, 37): db/000007.sst -read-at(53, 491): db/000007.sst +read-at(509, 53): db/000007.sst +read-at(472, 37): db/000007.sst +read-at(53, 419): db/000007.sst read-at(52, 27): db/000005.sst open: db/000005.sst read-at(0, 52): db/000005.sst diff --git a/testdata/compaction_delete_only_hints b/testdata/compaction_delete_only_hints index efb96610b8..cb613eb140 100644 --- a/testdata/compaction_delete_only_hints +++ b/testdata/compaction_delete_only_hints @@ -88,7 +88,7 @@ maybe-compact Deletion hints: (none) Compactions: - [JOB 100] compacted(delete-only) L2 [000005] (677B) Score=0.00 + L3 [000006] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s + [JOB 100] compacted(delete-only) L2 [000005] (605B) Score=0.00 + L3 [000006] (605B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s # Verify that compaction correctly handles the presence of multiple # overlapping hints which might delete a file multiple times. All of the @@ -127,7 +127,7 @@ maybe-compact Deletion hints: (none) Compactions: - [JOB 100] compacted(delete-only) L2 [000006] (677B) Score=0.00 + L3 [000007] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s + [JOB 100] compacted(delete-only) L2 [000006] (605B) Score=0.00 + L3 [000007] (605B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s # Test a range tombstone that is already compacted into L6. @@ -206,7 +206,7 @@ maybe-compact Deletion hints: (none) Compactions: - [JOB 100] compacted(delete-only) L2 [000005] (677B) Score=0.00 + L3 [000006] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s + [JOB 100] compacted(delete-only) L2 [000005] (605B) Score=0.00 + L3 [000006] (605B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s # A deletion hint present on an sstable in a higher level should NOT result in a # deletion-only compaction incorrectly removing an sstable in L6 following an @@ -255,7 +255,7 @@ L0.000001 a-z seqnums(tombstone=5-27, file-smallest=0, type=point-key-only) close-snapshot 10 ---- -[JOB 100] compacted(elision-only) L6 [000004] (742B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (663B), in 1.0s (2.0s total), output rate 663B/s +[JOB 100] compacted(elision-only) L6 [000004] (670B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (591B), in 1.0s (2.0s total), output rate 591B/s # The deletion hint was removed by the elision-only compaction. get-hints @@ -414,4 +414,4 @@ maybe-compact Deletion hints: (none) Compactions: - [JOB 100] compacted(delete-only) L6 [000006 000007 000008 000009 000011] (3.8KB) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s + [JOB 100] compacted(delete-only) L6 [000006 000007 000008 000009 000011] (3.5KB) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s diff --git a/testdata/compaction_picker_pick_file b/testdata/compaction_picker_pick_file index 6ecc838bd1..b19ff83fb5 100644 --- a/testdata/compaction_picker_pick_file +++ b/testdata/compaction_picker_pick_file @@ -16,9 +16,9 @@ L2 file-sizes ---- L1: - 000004:[b#11,1-c#11,1]: 669 bytes (669B) + 000004:[b#11,1-c#11,1]: 597 bytes (597B) L2: - 000005:[c#0,1-d#0,1]: 668 bytes (668B) + 000005:[c#0,1-d#0,1]: 596 bytes (596B) pick-file L1 ---- diff --git a/testdata/compaction_picker_scores b/testdata/compaction_picker_scores index 1139cf5595..56535784e6 100644 --- a/testdata/compaction_picker_scores +++ b/testdata/compaction_picker_scores @@ -24,7 +24,7 @@ L1 0B 0.0 L2 0B 0.0 L3 0B 0.0 L4 0B 0.0 -L5 729B 0.0 +L5 657B 0.0 L6 321KB - enable-table-stats @@ -37,7 +37,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 328519 +range-deletions-bytes-estimate: 328447 scores ---- @@ -47,7 +47,7 @@ L1 0B 0.0 L2 0B 0.0 L3 0B 0.0 L4 0B 0.0 -L5 729B 4.5 +L5 657B 4.5 L6 321KB - # Ensure that point deletions in a higher level result in a compensated level @@ -80,7 +80,7 @@ L1 0B 0.0 L2 0B 0.0 L3 0B 0.0 L4 0B 0.0 -L5 715B 0.0 +L5 643B 0.0 L6 321KB - enable-table-stats @@ -92,7 +92,7 @@ wait-pending-table-stats num-entries: 5 num-deletions: 5 num-range-key-sets: 0 -point-deletions-bytes-estimate: 164581 +point-deletions-bytes-estimate: 164509 range-deletions-bytes-estimate: 0 scores @@ -103,7 +103,7 @@ L1 0B 0.0 L2 0B 0.0 L3 0B 0.0 L4 0B 0.0 -L5 715B 2.3 +L5 643B 2.3 L6 321KB - # Run a similar test as above, but this time the table containing the DELs is @@ -145,7 +145,7 @@ wait-pending-table-stats num-entries: 5 num-deletions: 5 num-range-key-sets: 0 -point-deletions-bytes-estimate: 164605 +point-deletions-bytes-estimate: 164533 range-deletions-bytes-estimate: 0 maybe-compact diff --git a/testdata/compaction_read_triggered b/testdata/compaction_read_triggered index 51336f0d9e..d93ce77b18 100644 --- a/testdata/compaction_read_triggered +++ b/testdata/compaction_read_triggered @@ -20,7 +20,7 @@ show-read-compactions maybe-compact ---- -[JOB 100] compacted(read) L5 [000004] (647B) Score=0.00 + L6 [000005] (647B) Score=0.00 -> L6 [000006] (641B), in 1.0s (2.0s total), output rate 641B/s +[JOB 100] compacted(read) L5 [000004] (575B) Score=0.00 + L6 [000005] (575B) Score=0.00 -> L6 [000006] (569B), in 1.0s (2.0s total), output rate 569B/s show-read-compactions ---- @@ -81,7 +81,7 @@ show-read-compactions maybe-compact ---- -[JOB 100] compacted(read) L5 [000004] (647B) Score=0.00 + L6 [000005] (647B) Score=0.00 -> L6 [000006] (641B), in 1.0s (2.0s total), output rate 641B/s +[JOB 100] compacted(read) L5 [000004] (575B) Score=0.00 + L6 [000005] (575B) Score=0.00 -> L6 [000006] (569B), in 1.0s (2.0s total), output rate 569B/s show-read-compactions ---- diff --git a/testdata/compaction_tombstones b/testdata/compaction_tombstones index 795c0190dc..b2226bc0a7 100644 --- a/testdata/compaction_tombstones +++ b/testdata/compaction_tombstones @@ -41,7 +41,7 @@ range-deletions-bytes-estimate: 0 maybe-compact ---- -[JOB 100] compacted(elision-only) L6 [000004] (748B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s +[JOB 100] compacted(elision-only) L6 [000004] (676B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s # Test a table that straddles a snapshot. It should not be compacted. define snapshots=(50) auto-compactions=off @@ -80,12 +80,12 @@ wait-pending-table-stats num-entries: 2 num-deletions: 1 num-range-key-sets: 0 -point-deletions-bytes-estimate: 93 +point-deletions-bytes-estimate: 84 range-deletions-bytes-estimate: 0 maybe-compact ---- -[JOB 100] compacted(elision-only) L6 [000004] (714B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (663B), in 1.0s (2.0s total), output rate 663B/s +[JOB 100] compacted(elision-only) L6 [000004] (642B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (591B), in 1.0s (2.0s total), output rate 591B/s version ---- @@ -119,7 +119,7 @@ wait-pending-table-stats num-entries: 6 num-deletions: 2 num-range-key-sets: 0 -point-deletions-bytes-estimate: 42 +point-deletions-bytes-estimate: 39 range-deletions-bytes-estimate: 66 maybe-compact @@ -134,7 +134,7 @@ close-snapshot close-snapshot 103 ---- -[JOB 100] compacted(elision-only) L6 [000004] (892B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s +[JOB 100] compacted(elision-only) L6 [000004] (820B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s # Test a table that contains both deletions and non-deletions, but whose # non-deletions well outnumber its deletions. The table should not be @@ -152,7 +152,7 @@ wait-pending-table-stats num-entries: 11 num-deletions: 1 num-range-key-sets: 0 -point-deletions-bytes-estimate: 23 +point-deletions-bytes-estimate: 21 range-deletions-bytes-estimate: 0 close-snapshot @@ -208,7 +208,7 @@ range-deletions-bytes-estimate: 16492 maybe-compact ---- -[JOB 100] compacted(default) L5 [000004 000005] (26KB) Score=87.72 + L6 [000007] (17KB) Score=0.73 -> L6 [000009] (25KB), in 1.0s (2.0s total), output rate 25KB/s +[JOB 100] compacted(default) L5 [000004 000005] (25KB) Score=87.23 + L6 [000007] (17KB) Score=0.73 -> L6 [000009] (25KB), in 1.0s (2.0s total), output rate 25KB/s define level-max-bytes=(L5 : 1000) auto-compactions=off L5 @@ -233,7 +233,7 @@ wait-pending-table-stats num-entries: 3 num-deletions: 3 num-range-key-sets: 0 -point-deletions-bytes-estimate: 6858 +point-deletions-bytes-estimate: 6786 range-deletions-bytes-estimate: 0 # By plain file size, 000005 should be picked because it is larger and @@ -243,7 +243,7 @@ range-deletions-bytes-estimate: 0 maybe-compact ---- -[JOB 100] compacted(default) L5 [000004] (724B) Score=13.45 + L6 [000006] (13KB) Score=0.92 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s +[JOB 100] compacted(default) L5 [000004] (652B) Score=13.17 + L6 [000006] (13KB) Score=0.92 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s # A table containing only range keys is not eligible for elision. # RANGEKEYDEL or RANGEKEYUNSET. @@ -323,7 +323,7 @@ range-deletions-bytes-estimate: 41 maybe-compact ---- -[JOB 100] compacted(elision-only) L6 [000004] (894B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (669B), in 1.0s (2.0s total), output rate 669B/s +[JOB 100] compacted(elision-only) L6 [000004] (822B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (597B), in 1.0s (2.0s total), output rate 597B/s # Close the DB, asserting that the reference counts balance. close @@ -359,7 +359,7 @@ wait-pending-table-stats num-entries: 2 num-deletions: 1 num-range-key-sets: 0 -point-deletions-bytes-estimate: 2742 +point-deletions-bytes-estimate: 2713 range-deletions-bytes-estimate: 0 wait-pending-table-stats @@ -373,7 +373,7 @@ range-deletions-bytes-estimate: 8246 maybe-compact ---- -[JOB 100] compacted(default) L5 [000005] (741B) Score=11.82 + L6 [000007] (13KB) Score=1.05 -> L6 [000008] (4.7KB), in 1.0s (2.0s total), output rate 4.7KB/s +[JOB 100] compacted(default) L5 [000005] (669B) Score=11.60 + L6 [000007] (13KB) Score=1.06 -> L6 [000008] (4.6KB), in 1.0s (2.0s total), output rate 4.6KB/s # The same LSM as above. However, this time, with point tombstone weighting at # 2x, the table with the point tombstone (000004) will be selected as the @@ -402,7 +402,7 @@ wait-pending-table-stats num-entries: 2 num-deletions: 1 num-range-key-sets: 0 -point-deletions-bytes-estimate: 2742 +point-deletions-bytes-estimate: 2713 range-deletions-bytes-estimate: 0 wait-pending-table-stats @@ -416,4 +416,4 @@ range-deletions-bytes-estimate: 8246 maybe-compact ---- -[JOB 100] compacted(default) L5 [000005] (741B) Score=11.82 + L6 [000007] (13KB) Score=1.05 -> L6 [000008] (4.7KB), in 1.0s (2.0s total), output rate 4.7KB/s +[JOB 100] compacted(default) L5 [000005] (669B) Score=11.60 + L6 [000007] (13KB) Score=1.06 -> L6 [000008] (4.6KB), in 1.0s (2.0s total), output rate 4.6KB/s diff --git a/testdata/event_listener b/testdata/event_listener index aa135deac7..1cf535916c 100644 --- a/testdata/event_listener +++ b/testdata/event_listener @@ -75,7 +75,7 @@ close: db/marker.manifest.000002.MANIFEST-000006 remove: db/marker.manifest.000001.MANIFEST-000001 sync: db [JOB 3] MANIFEST created 000006 -[JOB 3] flushed 1 memtable (100B) to L0 [000005] (662B), in 1.0s (2.0s total), output rate 662B/s +[JOB 3] flushed 1 memtable (100B) to L0 [000005] (590B), in 1.0s (2.0s total), output rate 590B/s compact ---- @@ -99,18 +99,18 @@ close: db/marker.manifest.000003.MANIFEST-000009 remove: db/marker.manifest.000002.MANIFEST-000006 sync: db [JOB 5] MANIFEST created 000009 -[JOB 5] flushed 1 memtable (100B) to L0 [000008] (662B), in 1.0s (2.0s total), output rate 662B/s +[JOB 5] flushed 1 memtable (100B) to L0 [000008] (590B), in 1.0s (2.0s total), output rate 590B/s remove: db/MANIFEST-000001 [JOB 5] MANIFEST deleted 000001 -[JOB 6] compacting(default) L0 [000005 000008] (1.3KB) Score=0.00 + L6 [] (0B) Score=0.00; OverlappingRatio: Single 0.00, Multi 0.00 +[JOB 6] compacting(default) L0 [000005 000008] (1.2KB) Score=0.00 + L6 [] (0B) Score=0.00; OverlappingRatio: Single 0.00, Multi 0.00 open: db/000005.sst -read-at(609, 53): db/000005.sst -read-at(572, 37): db/000005.sst -read-at(53, 519): db/000005.sst +read-at(537, 53): db/000005.sst +read-at(500, 37): db/000005.sst +read-at(53, 447): db/000005.sst open: db/000008.sst -read-at(609, 53): db/000008.sst -read-at(572, 37): db/000008.sst -read-at(53, 519): db/000008.sst +read-at(537, 53): db/000008.sst +read-at(500, 37): db/000008.sst +read-at(53, 447): db/000008.sst read-at(26, 27): db/000005.sst open: db/000005.sst read-at(0, 26): db/000005.sst @@ -132,7 +132,7 @@ close: db/marker.manifest.000004.MANIFEST-000011 remove: db/marker.manifest.000003.MANIFEST-000009 sync: db [JOB 6] MANIFEST created 000011 -[JOB 6] compacted(default) L0 [000005 000008] (1.3KB) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000010] (662B), in 1.0s (3.0s total), output rate 662B/s +[JOB 6] compacted(default) L0 [000005 000008] (1.2KB) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000010] (590B), in 1.0s (3.0s total), output rate 590B/s close: db/000005.sst close: db/000008.sst remove: db/000005.sst @@ -167,7 +167,7 @@ close: db/marker.manifest.000005.MANIFEST-000014 remove: db/marker.manifest.000004.MANIFEST-000011 sync: db [JOB 8] MANIFEST created 000014 -[JOB 8] flushed 1 memtable (100B) to L0 [000013] (662B), in 1.0s (2.0s total), output rate 662B/s +[JOB 8] flushed 1 memtable (100B) to L0 [000013] (590B), in 1.0s (2.0s total), output rate 590B/s enable-file-deletions ---- @@ -177,9 +177,9 @@ remove: db/MANIFEST-000009 ingest ---- open: ext/0 -read-at(641, 53): ext/0 -read-at(604, 37): ext/0 -read-at(53, 551): ext/0 +read-at(569, 53): ext/0 +read-at(532, 37): ext/0 +read-at(53, 479): ext/0 read-at(26, 27): ext/0 read-at(0, 26): ext/0 close: ext/0 @@ -187,9 +187,9 @@ link: ext/0 -> db/000015.sst [JOB 10] ingesting: sstable created 000015 sync: db open: db/000013.sst -read-at(609, 53): db/000013.sst -read-at(572, 37): db/000013.sst -read-at(53, 519): db/000013.sst +read-at(537, 53): db/000013.sst +read-at(500, 37): db/000013.sst +read-at(53, 447): db/000013.sst read-at(26, 27): db/000013.sst read-at(0, 26): db/000013.sst create: db/MANIFEST-000016 @@ -203,32 +203,32 @@ sync: db remove: db/MANIFEST-000011 [JOB 10] MANIFEST deleted 000011 remove: ext/0 -[JOB 10] ingested L0:000015 (694B) +[JOB 10] ingested L0:000015 (622B) metrics ---- | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 2 1.3KB 0B 0 | 0.40 | 81B | 1 694B | 0 0B | 3 1.9KB | 0B | 2 24.5 + 0 | 2 1.2KB 0B 0 | 0.40 | 81B | 1 622B | 0 0B | 3 1.7KB | 0B | 2 21.9 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 1 662B 0B 0 | - | 1.3KB | 0 0B | 0 0B | 1 662B | 1.3KB | 1 0.5 -total | 3 2.0KB 0B 0 | - | 802B | 1 694B | 0 0B | 4 3.4KB | 1.3KB | 3 4.3 + 6 | 1 590B 0B 0 | - | 1.2KB | 0 0B | 0 0B | 1 590B | 1.2KB | 1 0.5 +total | 3 1.8KB 0B 0 | - | 730B | 1 622B | 0 0B | 4 3.0KB | 1.2KB | 3 4.2 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (27B) in: 48B written: 108B (125% overhead) Flushes: 3 -Compactions: 1 estimated debt: 2.0KB in progress: 0 (0B) +Compactions: 1 estimated debt: 1.8KB in progress: 0 (0B) default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (256KB) zombie: 1 (256KB) Zombie tables: 0 (0B) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 6 entries (1.1KB) hit rate: 0.0% -Table cache: 1 entries (800B) hit rate: 40.0% +Block cache: 6 entries (1002B) hit rate: 0.0% +Table cache: 1 entries (768B) hit rate: 40.0% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 0 @@ -243,16 +243,16 @@ ingest-flushable ---- sync-data: wal/000012.log open: ext/a -read-at(641, 53): ext/a -read-at(604, 37): ext/a -read-at(53, 551): ext/a +read-at(569, 53): ext/a +read-at(532, 37): ext/a +read-at(53, 479): ext/a read-at(26, 27): ext/a read-at(0, 26): ext/a close: ext/a open: ext/b -read-at(641, 53): ext/b -read-at(604, 37): ext/b -read-at(53, 551): ext/b +read-at(569, 53): ext/b +read-at(532, 37): ext/b +read-at(53, 479): ext/b read-at(26, 27): ext/b read-at(0, 26): ext/b close: ext/b @@ -274,7 +274,7 @@ sync: wal [JOB 13] WAL created 000020 remove: ext/a remove: ext/b -[JOB 11] ingested as flushable 000017 (694B), 000018 (694B) +[JOB 11] ingested as flushable 000017 (622B), 000018 (622B) sync-data: wal/000020.log close: wal/000020.log create: wal/000021.log @@ -287,7 +287,7 @@ sync-data: db/000022.sst close: db/000022.sst sync: db sync: db/MANIFEST-000016 -[JOB 15] flushed 1 memtable (100B) to L0 [000022] (662B), in 1.0s (2.0s total), output rate 662B/s +[JOB 15] flushed 1 memtable (100B) to L0 [000022] (590B), in 1.0s (2.0s total), output rate 590B/s [JOB 16] flushing 2 ingested tables create: db/MANIFEST-000023 close: db/MANIFEST-000016 @@ -297,7 +297,7 @@ close: db/marker.manifest.000007.MANIFEST-000023 remove: db/marker.manifest.000006.MANIFEST-000016 sync: db [JOB 16] MANIFEST created 000023 -[JOB 16] flushed 2 ingested flushables L0:000017 (694B) + L6:000018 (694B) in 1.0s (2.0s total), output rate 1.4KB/s +[JOB 16] flushed 2 ingested flushables L0:000017 (622B) + L6:000018 (622B) in 1.0s (2.0s total), output rate 1.2KB/s remove: db/MANIFEST-000014 [JOB 16] MANIFEST deleted 000014 [JOB 17] flushing 1 memtable (100B) to L0 @@ -309,30 +309,30 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 4 2.6KB 0B 0 | 0.80 | 81B | 2 1.4KB | 0 0B | 4 2.6KB | 0B | 4 32.7 + 0 | 4 2.4KB 0B 0 | 0.80 | 81B | 2 1.2KB | 0 0B | 4 2.3KB | 0B | 4 29.1 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 2 1.3KB 0B 0 | - | 1.3KB | 1 694B | 0 0B | 1 662B | 1.3KB | 1 0.5 -total | 6 4.0KB 0B 0 | - | 2.1KB | 3 2.0KB | 0 0B | 5 5.4KB | 1.3KB | 5 2.5 + 6 | 2 1.2KB 0B 0 | - | 1.2KB | 1 622B | 0 0B | 1 590B | 1.2KB | 1 0.5 +total | 6 3.6KB 0B 0 | - | 1.9KB | 3 1.8KB | 0 0B | 5 4.8KB | 1.2KB | 5 2.5 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (29B) in: 82B written: 110B (34% overhead) Flushes: 6 -Compactions: 1 estimated debt: 4.0KB in progress: 0 (0B) +Compactions: 1 estimated debt: 3.6KB in progress: 0 (0B) default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (512KB) zombie: 1 (512KB) Zombie tables: 0 (0B) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 12 entries (2.3KB) hit rate: 7.7% -Table cache: 1 entries (800B) hit rate: 50.0% +Block cache: 12 entries (2.0KB) hit rate: 7.7% +Table cache: 1 entries (768B) hit rate: 50.0% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 0 Filter utility: 0.0% -Ingestions: 1 as flushable: 1 (1.4KB in 2 tables) +Ingestions: 1 as flushable: 1 (1.2KB in 2 tables) sstables ---- diff --git a/testdata/ingest b/testdata/ingest index ff6ed9278e..e42b2b9cdb 100644 --- a/testdata/ingest +++ b/testdata/ingest @@ -40,8 +40,8 @@ level | tables size val-bl vtables | score | in | tables size | tables siz 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 1 673B 0B 0 | - | 0B | 1 673B | 0 0B | 0 0B | 0B | 1 0.0 -total | 1 673B 0B 0 | - | 673B | 1 673B | 0 0B | 0 673B | 0B | 1 1.0 + 6 | 1 601B 0B 0 | - | 0B | 1 601B | 0 0B | 0 0B | 0B | 1 0.0 +total | 1 601B 0B 0 | - | 601B | 1 601B | 0 0B | 0 601B | 0B | 1 1.0 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (0B) in: 0B written: 0B (0% overhead) Flushes: 0 @@ -51,8 +51,8 @@ MemTables: 1 (256KB) zombie: 0 (0B) Zombie tables: 0 (0B) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 6 entries (1.1KB) hit rate: 35.7% -Table cache: 1 entries (800B) hit rate: 50.0% +Block cache: 6 entries (1009B) hit rate: 35.7% +Table cache: 1 entries (768B) hit rate: 50.0% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 0 @@ -356,7 +356,7 @@ num-entries: 2 num-deletions: 2 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1374 +range-deletions-bytes-estimate: 1230 # A set operation takes precedence over a range deletion at the same # sequence number as can occur during ingestion. diff --git a/testdata/iter_histories/errors b/testdata/iter_histories/errors index b6e955afcd..cefdc0b682 100644 --- a/testdata/iter_histories/errors +++ b/testdata/iter_histories/errors @@ -80,10 +80,10 @@ layout filename=000004.sst 0 data (38) 43 index (35) 83 range-key (29) - 117 properties (645) - 767 meta-index (57) - 829 footer (53) - 882 EOF + 117 properties (573) + 695 meta-index (57) + 757 footer (53) + 810 EOF # Inject an error on the first `ReadAt` call on 000004.sst's range key block # (which is at offset 83). @@ -128,10 +128,10 @@ layout filename=000004.sst 139 data (22) 166 data (22) 193 index (113) - 311 properties (643) - 959 meta-index (33) - 997 footer (53) - 1050 EOF + 311 properties (571) + 887 meta-index (33) + 925 footer (53) + 978 EOF reopen auto-compactions=off enable-table-stats=false inject-errors=((ErrInjected (And (PathMatch "000004.sst") (OpFileReadAt 0)))) ---- @@ -182,10 +182,10 @@ layout filename=000004.sst 81 data (22) 108 data (22) 135 index (84) - 224 properties (567) - 796 meta-index (33) - 834 footer (53) - 887 EOF + 224 properties (495) + 724 meta-index (33) + 762 footer (53) + 815 EOF # NB: Block offset to key contained: # 0 -> a, 27 -> b, 54 -> c, 81 -> d, 108 -> e @@ -243,10 +243,10 @@ layout filename=000004.sst 54 data (22) 81 data (22) 108 index (71) - 184 properties (642) - 831 meta-index (33) - 869 footer (53) - 922 EOF + 184 properties (570) + 759 meta-index (33) + 797 footer (53) + 850 EOF reopen auto-compactions=off enable-table-stats=false inject-errors=((ErrInjected (And (PathMatch "000004.sst") (OpFileReadAt 54)))) diff --git a/testdata/manual_compaction_file_boundaries_delsized b/testdata/manual_compaction_file_boundaries_delsized index 61a2b313f5..866cc5d85e 100644 --- a/testdata/manual_compaction_file_boundaries_delsized +++ b/testdata/manual_compaction_file_boundaries_delsized @@ -290,60 +290,60 @@ compact a-zz L1 file-sizes ---- L2: - 000095:[a#101,1-az@1#129,1]: 7528 bytes (7.4KB) - 000096:[b@1#130,1-bz@1#156,1]: 6520 bytes (6.4KB) - 000097:[c@1#157,1-cz@1#183,1]: 6520 bytes (6.4KB) - 000098:[d@1#184,1-dz@1#210,1]: 6520 bytes (6.4KB) - 000099:[e@1#211,1-ez@1#237,1]: 6520 bytes (6.4KB) - 000100:[f@1#238,1-fz@1#264,1]: 6520 bytes (6.4KB) - 000101:[g@1#265,1-gz@1#291,1]: 6520 bytes (6.4KB) - 000102:[h@1#292,1-hz@1#318,1]: 6520 bytes (6.4KB) - 000103:[i@1#319,1-iz@1#345,1]: 6520 bytes (6.4KB) - 000104:[j@1#346,1-jz@1#372,1]: 6520 bytes (6.4KB) - 000105:[k@1#373,1-kz@1#399,1]: 6520 bytes (6.4KB) - 000106:[l@1#400,1-lz@1#426,1]: 6520 bytes (6.4KB) - 000107:[m@1#427,1-mz@1#453,1]: 6520 bytes (6.4KB) - 000108:[n@1#454,1-nz@1#480,1]: 6520 bytes (6.4KB) - 000109:[o@1#481,1-oz@1#507,1]: 6520 bytes (6.4KB) - 000110:[p@1#508,1-pz@1#534,1]: 6520 bytes (6.4KB) - 000111:[q@1#535,1-qz@1#561,1]: 6519 bytes (6.4KB) - 000112:[r@1#562,1-rz@1#588,1]: 6520 bytes (6.4KB) - 000113:[s@1#589,1-sz@1#615,1]: 6520 bytes (6.4KB) - 000114:[t@1#616,1-tz@1#642,1]: 6520 bytes (6.4KB) - 000115:[u@1#643,1-uz@1#669,1]: 6520 bytes (6.4KB) - 000116:[v@1#670,1-vz@1#696,1]: 6520 bytes (6.4KB) - 000117:[w@1#697,1-wz@1#723,1]: 6520 bytes (6.4KB) - 000118:[x@1#724,1-xz@1#750,1]: 6520 bytes (6.4KB) - 000119:[y@1#751,1-yz@1#777,1]: 6520 bytes (6.4KB) - 000120:[z#102,1-zr@1#796,1]: 5800 bytes (5.7KB) - 000121:[zs@1#797,1-zz@1#804,1]: 2382 bytes (2.3KB) + 000095:[a#101,1-az@1#129,1]: 7456 bytes (7.3KB) + 000096:[b@1#130,1-bz@1#156,1]: 6448 bytes (6.3KB) + 000097:[c@1#157,1-cz@1#183,1]: 6448 bytes (6.3KB) + 000098:[d@1#184,1-dz@1#210,1]: 6448 bytes (6.3KB) + 000099:[e@1#211,1-ez@1#237,1]: 6448 bytes (6.3KB) + 000100:[f@1#238,1-fz@1#264,1]: 6448 bytes (6.3KB) + 000101:[g@1#265,1-gz@1#291,1]: 6448 bytes (6.3KB) + 000102:[h@1#292,1-hz@1#318,1]: 6448 bytes (6.3KB) + 000103:[i@1#319,1-iz@1#345,1]: 6448 bytes (6.3KB) + 000104:[j@1#346,1-jz@1#372,1]: 6448 bytes (6.3KB) + 000105:[k@1#373,1-kz@1#399,1]: 6448 bytes (6.3KB) + 000106:[l@1#400,1-lz@1#426,1]: 6448 bytes (6.3KB) + 000107:[m@1#427,1-mz@1#453,1]: 6448 bytes (6.3KB) + 000108:[n@1#454,1-nz@1#480,1]: 6448 bytes (6.3KB) + 000109:[o@1#481,1-oz@1#507,1]: 6448 bytes (6.3KB) + 000110:[p@1#508,1-pz@1#534,1]: 6448 bytes (6.3KB) + 000111:[q@1#535,1-qz@1#561,1]: 6447 bytes (6.3KB) + 000112:[r@1#562,1-rz@1#588,1]: 6448 bytes (6.3KB) + 000113:[s@1#589,1-sz@1#615,1]: 6448 bytes (6.3KB) + 000114:[t@1#616,1-tz@1#642,1]: 6448 bytes (6.3KB) + 000115:[u@1#643,1-uz@1#669,1]: 6448 bytes (6.3KB) + 000116:[v@1#670,1-vz@1#696,1]: 6448 bytes (6.3KB) + 000117:[w@1#697,1-wz@1#723,1]: 6448 bytes (6.3KB) + 000118:[x@1#724,1-xz@1#750,1]: 6448 bytes (6.3KB) + 000119:[y@1#751,1-yz@1#777,1]: 6448 bytes (6.3KB) + 000120:[z#102,1-zr@1#796,1]: 5728 bytes (5.6KB) + 000121:[zs@1#797,1-zz@1#804,1]: 2310 bytes (2.3KB) L3: - 000005:[a#1,1-a#1,1]: 10667 bytes (10KB) - 000006:[b#2,1-b#2,1]: 10667 bytes (10KB) - 000007:[c#3,1-c#3,1]: 10667 bytes (10KB) - 000008:[d#4,1-d#4,1]: 10667 bytes (10KB) - 000009:[e#5,1-e#5,1]: 10667 bytes (10KB) - 000010:[f#6,1-f#6,1]: 10667 bytes (10KB) - 000011:[g#7,1-g#7,1]: 10667 bytes (10KB) - 000012:[h#8,1-h#8,1]: 10667 bytes (10KB) - 000013:[i#9,1-i#9,1]: 10667 bytes (10KB) - 000014:[j#10,1-j#10,1]: 10667 bytes (10KB) - 000015:[k#11,1-k#11,1]: 10667 bytes (10KB) - 000016:[l#12,1-l#12,1]: 10667 bytes (10KB) - 000017:[m#13,1-m#13,1]: 10667 bytes (10KB) - 000018:[n#14,1-n#14,1]: 10667 bytes (10KB) - 000019:[o#15,1-o#15,1]: 10667 bytes (10KB) - 000020:[p#16,1-p#16,1]: 10667 bytes (10KB) - 000021:[q#17,1-q#17,1]: 10667 bytes (10KB) - 000022:[r#18,1-r#18,1]: 10667 bytes (10KB) - 000023:[s#19,1-s#19,1]: 10667 bytes (10KB) - 000024:[t#20,1-t#20,1]: 10667 bytes (10KB) - 000025:[u#21,1-u#21,1]: 10667 bytes (10KB) - 000026:[v#22,1-v#22,1]: 10667 bytes (10KB) - 000027:[w#23,1-w#23,1]: 10667 bytes (10KB) - 000028:[x#24,1-x#24,1]: 10667 bytes (10KB) - 000029:[y#25,1-y#25,1]: 10667 bytes (10KB) - 000030:[z#26,1-z#26,1]: 10667 bytes (10KB) + 000005:[a#1,1-a#1,1]: 10595 bytes (10KB) + 000006:[b#2,1-b#2,1]: 10595 bytes (10KB) + 000007:[c#3,1-c#3,1]: 10595 bytes (10KB) + 000008:[d#4,1-d#4,1]: 10595 bytes (10KB) + 000009:[e#5,1-e#5,1]: 10595 bytes (10KB) + 000010:[f#6,1-f#6,1]: 10595 bytes (10KB) + 000011:[g#7,1-g#7,1]: 10595 bytes (10KB) + 000012:[h#8,1-h#8,1]: 10595 bytes (10KB) + 000013:[i#9,1-i#9,1]: 10595 bytes (10KB) + 000014:[j#10,1-j#10,1]: 10595 bytes (10KB) + 000015:[k#11,1-k#11,1]: 10595 bytes (10KB) + 000016:[l#12,1-l#12,1]: 10595 bytes (10KB) + 000017:[m#13,1-m#13,1]: 10595 bytes (10KB) + 000018:[n#14,1-n#14,1]: 10595 bytes (10KB) + 000019:[o#15,1-o#15,1]: 10595 bytes (10KB) + 000020:[p#16,1-p#16,1]: 10595 bytes (10KB) + 000021:[q#17,1-q#17,1]: 10595 bytes (10KB) + 000022:[r#18,1-r#18,1]: 10595 bytes (10KB) + 000023:[s#19,1-s#19,1]: 10595 bytes (10KB) + 000024:[t#20,1-t#20,1]: 10595 bytes (10KB) + 000025:[u#21,1-u#21,1]: 10595 bytes (10KB) + 000026:[v#22,1-v#22,1]: 10595 bytes (10KB) + 000027:[w#23,1-w#23,1]: 10595 bytes (10KB) + 000028:[x#24,1-x#24,1]: 10595 bytes (10KB) + 000029:[y#25,1-y#25,1]: 10595 bytes (10KB) + 000030:[z#26,1-z#26,1]: 10595 bytes (10KB) # Test a scenario where there exists a grandparent file (in L3), but the L1->L2 # compaction doesn't reach it until late in the compaction. The output file @@ -399,11 +399,11 @@ compact a-zz L1 file-sizes ---- L2: - 000007:[a#201,1-j#210,1]: 10849 bytes (11KB) - 000008:[k#211,1-o#215,1]: 5756 bytes (5.6KB) - 000009:[z#102,1-z#102,1]: 672 bytes (672B) + 000007:[a#201,1-j#210,1]: 10777 bytes (10KB) + 000008:[k#211,1-o#215,1]: 5684 bytes (5.6KB) + 000009:[z#102,1-z#102,1]: 600 bytes (600B) L3: - 000006:[m#1,1-m#1,1]: 10667 bytes (10KB) + 000006:[m#1,1-m#1,1]: 10595 bytes (10KB) # Test the file-size splitter's adaptive tolerance for early-splitting at a # grandparent boundary. The L1->L2 compaction has many opportunities to split at @@ -501,20 +501,20 @@ compact a-zz L1 file-sizes ---- L2: - 000019:[a#201,1-e#205,1]: 5756 bytes (5.6KB) - 000020:[f#206,1-l#212,1]: 7784 bytes (7.6KB) - 000021:[m#213,1-z#102,1]: 3718 bytes (3.6KB) + 000019:[a#201,1-e#205,1]: 5684 bytes (5.6KB) + 000020:[f#206,1-l#212,1]: 7712 bytes (7.5KB) + 000021:[m#213,1-z#102,1]: 3646 bytes (3.6KB) L3: - 000006:[a#1,1-a#1,1]: 1667 bytes (1.6KB) - 000007:[ab#2,1-ab#2,1]: 1668 bytes (1.6KB) - 000008:[ac#3,1-ac#3,1]: 1668 bytes (1.6KB) - 000009:[ad#4,1-ad#4,1]: 1668 bytes (1.6KB) - 000010:[ae#5,1-ae#5,1]: 1668 bytes (1.6KB) - 000011:[af#6,1-af#6,1]: 1668 bytes (1.6KB) - 000012:[ag#7,1-ag#7,1]: 1668 bytes (1.6KB) - 000013:[ah#8,1-ah#8,1]: 1668 bytes (1.6KB) - 000014:[c#9,1-c#9,1]: 1667 bytes (1.6KB) - 000015:[d#10,1-d#10,1]: 1667 bytes (1.6KB) - 000016:[e#11,1-e#11,1]: 1667 bytes (1.6KB) - 000017:[f#12,1-f#12,1]: 1667 bytes (1.6KB) - 000018:[m#13,1-m#13,1]: 1667 bytes (1.6KB) + 000006:[a#1,1-a#1,1]: 1595 bytes (1.6KB) + 000007:[ab#2,1-ab#2,1]: 1596 bytes (1.6KB) + 000008:[ac#3,1-ac#3,1]: 1596 bytes (1.6KB) + 000009:[ad#4,1-ad#4,1]: 1596 bytes (1.6KB) + 000010:[ae#5,1-ae#5,1]: 1596 bytes (1.6KB) + 000011:[af#6,1-af#6,1]: 1596 bytes (1.6KB) + 000012:[ag#7,1-ag#7,1]: 1596 bytes (1.6KB) + 000013:[ah#8,1-ah#8,1]: 1596 bytes (1.6KB) + 000014:[c#9,1-c#9,1]: 1595 bytes (1.6KB) + 000015:[d#10,1-d#10,1]: 1595 bytes (1.6KB) + 000016:[e#11,1-e#11,1]: 1595 bytes (1.6KB) + 000017:[f#12,1-f#12,1]: 1595 bytes (1.6KB) + 000018:[m#13,1-m#13,1]: 1595 bytes (1.6KB) diff --git a/testdata/manual_compaction_set_with_del_sstable_Pebblev4 b/testdata/manual_compaction_set_with_del_sstable_Pebblev4 index 19bed7b1ba..b40b23ef25 100644 --- a/testdata/manual_compaction_set_with_del_sstable_Pebblev4 +++ b/testdata/manual_compaction_set_with_del_sstable_Pebblev4 @@ -88,7 +88,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1334 +range-deletions-bytes-estimate: 1190 compact a-e L1 ---- @@ -106,7 +106,7 @@ num-entries: 2 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 667 +range-deletions-bytes-estimate: 595 # Same as above, except range tombstone covers multiple grandparent file boundaries. diff --git a/testdata/marked_for_compaction b/testdata/marked_for_compaction index 25a6181872..c9c9ae0797 100644 --- a/testdata/marked_for_compaction +++ b/testdata/marked_for_compaction @@ -20,8 +20,8 @@ marked L0.000004 maybe-compact ---- -[JOB 100] compacted(rewrite) L1 [000005] (670B) Score=0.00 + L1 [] (0B) Score=0.00 -> L1 [000006] (670B), in 1.0s (2.0s total), output rate 670B/s -[JOB 100] compacted(rewrite) L0 [000004] (665B) Score=0.00 + L0 [] (0B) Score=0.00 -> L0 [000007] (665B), in 1.0s (2.0s total), output rate 665B/s +[JOB 100] compacted(rewrite) L1 [000005] (598B) Score=0.00 + L1 [] (0B) Score=0.00 -> L1 [000006] (598B), in 1.0s (2.0s total), output rate 598B/s +[JOB 100] compacted(rewrite) L0 [000004] (593B) Score=0.00 + L0 [] (0B) Score=0.00 -> L0 [000007] (593B), in 1.0s (2.0s total), output rate 593B/s 0.0: 000007:[c#11,SET-c#11,SET] seqnums:[11-11] points:[c#11,SET-c#11,SET] 1: diff --git a/testdata/metrics b/testdata/metrics index a376b00478..acd53062de 100644 --- a/testdata/metrics +++ b/testdata/metrics @@ -50,14 +50,14 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 1 661B 0B 0 | 0.25 | 28B | 0 0B | 0 0B | 1 661B | 0B | 1 23.6 + 0 | 1 589B 0B 0 | 0.25 | 28B | 0 0B | 0 0B | 1 589B | 0B | 1 21.0 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 6 | 0 0B 0B 0 | - | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 -total | 1 661B 0B 0 | - | 56B | 0 0B | 0 0B | 1 717B | 0B | 1 12.8 +total | 1 589B 0B 0 | - | 56B | 0 0B | 0 0B | 1 645B | 0B | 1 11.5 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (28B) in: 17B written: 56B (229% overhead) Flushes: 1 @@ -67,8 +67,8 @@ MemTables: 1 (256KB) zombie: 1 (256KB) Zombie tables: 0 (0B) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 3 entries (556B) hit rate: 0.0% -Table cache: 1 entries (800B) hit rate: 0.0% +Block cache: 3 entries (484B) hit rate: 0.0% +Table cache: 1 entries (768B) hit rate: 0.0% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 1 @@ -104,25 +104,25 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 0 0B 0B 0 | 0.00 | 56B | 0 0B | 0 0B | 2 1.3KB | 0B | 0 23.6 + 0 | 0 0B 0B 0 | 0.00 | 56B | 0 0B | 0 0B | 2 1.2KB | 0B | 0 21.0 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 1 667B 0B 0 | - | 1.3KB | 0 0B | 0 0B | 1 667B | 1.3KB | 1 0.5 -total | 1 667B 0B 0 | - | 84B | 0 0B | 0 0B | 3 2.0KB | 1.3KB | 1 24.7 + 6 | 1 595B 0B 0 | - | 1.2KB | 0 0B | 0 0B | 1 595B | 1.2KB | 1 0.5 +total | 1 595B 0B 0 | - | 84B | 0 0B | 0 0B | 3 1.8KB | 1.2KB | 1 22.1 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (28B) in: 34B written: 84B (147% overhead) Flushes: 2 Compactions: 1 estimated debt: 0B in progress: 0 (0B) default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (256KB) zombie: 2 (512KB) -Zombie tables: 2 (1.3KB) +Zombie tables: 2 (1.2KB) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 5 entries (1.1KB) hit rate: 33.3% -Table cache: 2 entries (1.6KB) hit rate: 66.7% +Block cache: 5 entries (946B) hit rate: 33.3% +Table cache: 2 entries (1.5KB) hit rate: 66.7% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 2 @@ -133,7 +133,7 @@ Iter category stats: disk-usage ---- -3.4KB +3.1KB # Closing iter a will release one of the zombie memtables. @@ -145,25 +145,25 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 0 0B 0B 0 | 0.00 | 56B | 0 0B | 0 0B | 2 1.3KB | 0B | 0 23.6 + 0 | 0 0B 0B 0 | 0.00 | 56B | 0 0B | 0 0B | 2 1.2KB | 0B | 0 21.0 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 1 667B 0B 0 | - | 1.3KB | 0 0B | 0 0B | 1 667B | 1.3KB | 1 0.5 -total | 1 667B 0B 0 | - | 84B | 0 0B | 0 0B | 3 2.0KB | 1.3KB | 1 24.7 + 6 | 1 595B 0B 0 | - | 1.2KB | 0 0B | 0 0B | 1 595B | 1.2KB | 1 0.5 +total | 1 595B 0B 0 | - | 84B | 0 0B | 0 0B | 3 1.8KB | 1.2KB | 1 22.1 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (28B) in: 34B written: 84B (147% overhead) Flushes: 2 Compactions: 1 estimated debt: 0B in progress: 0 (0B) default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (256KB) zombie: 2 (512KB) -Zombie tables: 2 (1.3KB) +Zombie tables: 2 (1.2KB) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 5 entries (1.1KB) hit rate: 33.3% -Table cache: 2 entries (1.6KB) hit rate: 66.7% +Block cache: 5 entries (946B) hit rate: 33.3% +Table cache: 2 entries (1.5KB) hit rate: 66.7% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 2 @@ -183,25 +183,25 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 0 0B 0B 0 | 0.00 | 56B | 0 0B | 0 0B | 2 1.3KB | 0B | 0 23.6 + 0 | 0 0B 0B 0 | 0.00 | 56B | 0 0B | 0 0B | 2 1.2KB | 0B | 0 21.0 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 1 667B 0B 0 | - | 1.3KB | 0 0B | 0 0B | 1 667B | 1.3KB | 1 0.5 -total | 1 667B 0B 0 | - | 84B | 0 0B | 0 0B | 3 2.0KB | 1.3KB | 1 24.7 + 6 | 1 595B 0B 0 | - | 1.2KB | 0 0B | 0 0B | 1 595B | 1.2KB | 1 0.5 +total | 1 595B 0B 0 | - | 84B | 0 0B | 0 0B | 3 1.8KB | 1.2KB | 1 22.1 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (28B) in: 34B written: 84B (147% overhead) Flushes: 2 Compactions: 1 estimated debt: 0B in progress: 0 (0B) default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (256KB) zombie: 2 (512KB) -Zombie tables: 1 (661B) +Zombie tables: 1 (589B) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 3 entries (556B) hit rate: 33.3% -Table cache: 1 entries (800B) hit rate: 66.7% +Block cache: 3 entries (484B) hit rate: 33.3% +Table cache: 1 entries (768B) hit rate: 66.7% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 1 @@ -213,7 +213,7 @@ Iter category stats: disk-usage ---- -2.7KB +2.6KB # Closing iter b will release the last zombie sstable and the last zombie memtable. @@ -225,14 +225,14 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 0 0B 0B 0 | 0.00 | 56B | 0 0B | 0 0B | 2 1.3KB | 0B | 0 23.6 + 0 | 0 0B 0B 0 | 0.00 | 56B | 0 0B | 0 0B | 2 1.2KB | 0B | 0 21.0 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 1 667B 0B 0 | - | 1.3KB | 0 0B | 0 0B | 1 667B | 1.3KB | 1 0.5 -total | 1 667B 0B 0 | - | 84B | 0 0B | 0 0B | 3 2.0KB | 1.3KB | 1 24.7 + 6 | 1 595B 0B 0 | - | 1.2KB | 0 0B | 0 0B | 1 595B | 1.2KB | 1 0.5 +total | 1 595B 0B 0 | - | 84B | 0 0B | 0 0B | 3 1.8KB | 1.2KB | 1 22.1 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (28B) in: 34B written: 84B (147% overhead) Flushes: 2 @@ -256,7 +256,7 @@ Iter category stats: disk-usage ---- -2.1KB +2.0KB additional-metrics ---- @@ -294,18 +294,18 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 3 2.2KB 38B 0 | 0.25 | 149B | 0 0B | 0 0B | 5 3.5KB | 0B | 1 24.2 + 0 | 3 2.0KB 38B 0 | 0.25 | 149B | 0 0B | 0 0B | 5 3.2KB | 0B | 1 21.8 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 1 667B 0B 0 | - | 1.3KB | 0 0B | 0 0B | 1 667B | 1.3KB | 1 0.5 -total | 4 2.9KB 38B 0 | - | 242B | 0 0B | 0 0B | 6 4.4KB | 1.3KB | 2 18.6 + 6 | 1 595B 0B 0 | - | 1.2KB | 0 0B | 0 0B | 1 595B | 1.2KB | 1 0.5 +total | 4 2.6KB 38B 0 | - | 242B | 0 0B | 0 0B | 6 4.0KB | 1.2KB | 2 16.9 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (93B) in: 116B written: 242B (109% overhead) Flushes: 3 -Compactions: 1 estimated debt: 2.9KB in progress: 0 (0B) +Compactions: 1 estimated debt: 2.6KB in progress: 0 (0B) default: 1 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (256KB) zombie: 1 (256KB) Zombie tables: 0 (0B) @@ -347,14 +347,14 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 0 0B 0B 0 | 0.00 | 149B | 0 0B | 0 0B | 5 3.5KB | 0B | 0 24.2 + 0 | 0 0B 0B 0 | 0.00 | 149B | 0 0B | 0 0B | 5 3.2KB | 0B | 0 21.8 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 3 2.2KB 41B 0 | - | 3.5KB | 0 0B | 0 0B | 3 2.2KB | 3.5KB | 1 0.6 -total | 3 2.2KB 41B 0 | - | 242B | 0 0B | 0 0B | 8 6.0KB | 3.5KB | 1 25.3 + 6 | 3 2.0KB 41B 0 | - | 3.2KB | 0 0B | 0 0B | 3 2.0KB | 3.2KB | 1 0.6 +total | 3 2.0KB 41B 0 | - | 242B | 0 0B | 0 0B | 8 5.4KB | 3.2KB | 1 23.0 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (93B) in: 116B written: 242B (109% overhead) Flushes: 3 @@ -449,30 +449,30 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 4 2.7KB 0B 0 | 0.50 | 149B | 3 2.0KB | 0 0B | 6 4.2KB | 0B | 2 28.8 + 0 | 4 2.4KB 0B 0 | 0.50 | 149B | 3 1.8KB | 0 0B | 6 3.8KB | 0B | 2 25.9 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 3 2.2KB 41B 0 | - | 3.5KB | 0 0B | 0 0B | 3 2.2KB | 3.5KB | 1 0.6 -total | 7 4.9KB 41B 0 | - | 2.2KB | 3 2.0KB | 0 0B | 9 8.6KB | 3.5KB | 3 3.9 + 6 | 3 2.0KB 41B 0 | - | 3.2KB | 0 0B | 0 0B | 3 2.0KB | 3.2KB | 1 0.6 +total | 7 4.4KB 41B 0 | - | 2.0KB | 3 1.8KB | 0 0B | 9 7.8KB | 3.2KB | 3 3.9 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (26B) in: 176B written: 175B (-1% overhead) Flushes: 8 -Compactions: 2 estimated debt: 4.9KB in progress: 0 (0B) +Compactions: 2 estimated debt: 4.4KB in progress: 0 (0B) default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (1.0MB) zombie: 1 (1.0MB) Zombie tables: 0 (0B) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 12 entries (2.3KB) hit rate: 16.7% -Table cache: 1 entries (800B) hit rate: 60.0% +Block cache: 12 entries (2.0KB) hit rate: 16.7% +Table cache: 1 entries (768B) hit rate: 60.0% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 0 Filter utility: 0.0% -Ingestions: 0 as flushable: 2 (2.0KB in 3 tables) +Ingestions: 0 as flushable: 2 (1.8KB in 3 tables) Iter category stats: b, latency: {BlockBytes:44 BlockBytesInCache:0 BlockReadDuration:10ms} c, non-latency: {BlockBytes:44 BlockBytesInCache:44 BlockReadDuration:0s} @@ -510,30 +510,30 @@ metrics | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 7 4.7KB 0B 0 | 0.50 | 207B | 3 2.0KB | 0 0B | 9 6.2KB | 0B | 2 30.5 + 0 | 7 4.2KB 0B 0 | 0.50 | 207B | 3 1.8KB | 0 0B | 9 5.5KB | 0B | 2 27.4 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 3 2.2KB 41B 0 | - | 3.5KB | 0 0B | 0 0B | 3 2.2KB | 3.5KB | 1 0.6 -total | 10 6.9KB 41B 0 | - | 2.3KB | 3 2.0KB | 0 0B | 12 11KB | 3.5KB | 3 4.7 + 6 | 3 2.0KB 41B 0 | - | 3.2KB | 0 0B | 0 0B | 3 2.0KB | 3.2KB | 1 0.6 +total | 10 6.2KB 41B 0 | - | 2.1KB | 3 1.8KB | 0 0B | 12 9.6KB | 3.2KB | 3 4.6 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (58B) in: 223B written: 265B (19% overhead) Flushes: 9 -Compactions: 2 estimated debt: 6.9KB in progress: 0 (0B) +Compactions: 2 estimated debt: 6.2KB in progress: 0 (0B) default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (1.0MB) zombie: 1 (1.0MB) Zombie tables: 0 (0B) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 12 entries (2.3KB) hit rate: 16.7% -Table cache: 1 entries (800B) hit rate: 60.0% +Block cache: 12 entries (2.0KB) hit rate: 16.7% +Table cache: 1 entries (768B) hit rate: 60.0% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 0 Filter utility: 0.0% -Ingestions: 0 as flushable: 2 (2.0KB in 3 tables) +Ingestions: 0 as flushable: 2 (1.8KB in 3 tables) Iter category stats: b, latency: {BlockBytes:44 BlockBytesInCache:0 BlockReadDuration:10ms} c, non-latency: {BlockBytes:44 BlockBytesInCache:44 BlockReadDuration:0s} @@ -575,7 +575,7 @@ num-virtual 0 virtual-size ---- 2 -1.3KB +1.2KB 2 2 102B @@ -585,30 +585,30 @@ metrics zero-cache-hits-misses | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 7 3.4KB 0B 2 | 0.50 | 207B | 3 2.0KB | 0 0B | 9 6.2KB | 0B | 2 30.5 + 0 | 7 3.1KB 0B 2 | 0.50 | 207B | 3 1.8KB | 0 0B | 9 5.5KB | 0B | 2 27.4 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 4 2.9KB 41B 0 | - | 3.5KB | 1 693B | 0 0B | 3 2.2KB | 3.5KB | 1 0.6 -total | 11 6.3KB 41B 2 | - | 3.0KB | 4 2.7KB | 0 0B | 12 11KB | 3.5KB | 3 3.8 + 6 | 4 2.6KB 41B 0 | - | 3.2KB | 1 621B | 0 0B | 3 2.0KB | 3.2KB | 1 0.6 +total | 11 5.7KB 41B 2 | - | 2.7KB | 4 2.4KB | 0 0B | 12 10KB | 3.2KB | 3 3.8 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (58B) in: 223B written: 265B (19% overhead) Flushes: 9 -Compactions: 2 estimated debt: 6.3KB in progress: 0 (0B) +Compactions: 2 estimated debt: 5.7KB in progress: 0 (0B) default: 2 delete: 0 elision: 0 move: 0 read: 0 rewrite: 0 multi-level: 0 MemTables: 1 (1.0MB) zombie: 1 (1.0MB) Zombie tables: 0 (0B) -Backing tables: 2 (1.3KB) +Backing tables: 2 (1.2KB) Virtual tables: 2 (102B) -Block cache: 21 entries (4.0KB) hit rate: 0.0% +Block cache: 21 entries (3.5KB) hit rate: 0.0% Table cache: 3 entries (2.3KB) hit rate: 0.0% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 0 Filter utility: 0.0% -Ingestions: 1 as flushable: 2 (2.0KB in 3 tables) +Ingestions: 1 as flushable: 2 (1.8KB in 3 tables) Iter category stats: b, latency: {BlockBytes:44 BlockBytesInCache:0 BlockReadDuration:10ms} c, non-latency: {BlockBytes:44 BlockBytesInCache:44 BlockReadDuration:0s} @@ -651,7 +651,7 @@ num-virtual 0 virtual-size ---- 2 -1.3KB +1.2KB 2 2 102B @@ -685,14 +685,14 @@ metrics zero-cache-hits-misses | | | | ingested | moved | written | | amp level | tables size val-bl vtables | score | in | tables size | tables size | tables size | read | r w ------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+--------- - 0 | 0 0B 0B 0 | 0.00 | 207B | 3 2.0KB | 0 0B | 9 6.2KB | 0B | 0 30.5 + 0 | 0 0B 0B 0 | 0.00 | 207B | 3 1.8KB | 0 0B | 9 5.5KB | 0B | 0 27.4 1 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 2 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 3 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 4 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 5 | 0 0B 0B 0 | 0.00 | 0B | 0 0B | 0 0B | 0 0B | 0B | 0 0.0 - 6 | 6 4.3KB 41B 0 | - | 7.0KB | 2 1.4KB | 0 0B | 4 2.9KB | 7.0KB | 1 0.4 -total | 6 4.3KB 41B 0 | - | 3.6KB | 5 3.4KB | 0 0B | 13 13KB | 7.0KB | 1 3.5 + 6 | 6 3.9KB 41B 0 | - | 6.3KB | 2 1.2KB | 0 0B | 4 2.6KB | 6.3KB | 1 0.4 +total | 6 3.9KB 41B 0 | - | 3.3KB | 5 3.0KB | 0 0B | 13 12KB | 6.3KB | 1 3.5 ------------------------------------------------------------------------------------------------------------------- WAL: 1 files (58B) in: 223B written: 265B (19% overhead) Flushes: 9 @@ -702,13 +702,13 @@ MemTables: 1 (1.0MB) zombie: 1 (1.0MB) Zombie tables: 0 (0B) Backing tables: 0 (0B) Virtual tables: 0 (0B) -Block cache: 6 entries (1.2KB) hit rate: 0.0% +Block cache: 6 entries (1.0KB) hit rate: 0.0% Table cache: 0 entries (0B) hit rate: 0.0% Secondary cache: 0 entries (0B) hit rate: 0.0% Snapshots: 0 earliest seq num: 0 Table iters: 0 Filter utility: 0.0% -Ingestions: 2 as flushable: 2 (2.0KB in 3 tables) +Ingestions: 2 as flushable: 2 (1.8KB in 3 tables) Iter category stats: b, latency: {BlockBytes:44 BlockBytesInCache:0 BlockReadDuration:10ms} c, non-latency: {BlockBytes:44 BlockBytesInCache:44 BlockReadDuration:0s} diff --git a/testdata/range_del b/testdata/range_del index 19813d6b19..ba1fffa20e 100644 --- a/testdata/range_del +++ b/testdata/range_del @@ -1241,7 +1241,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 699 +range-deletions-bytes-estimate: 627 wait-pending-table-stats 000004 @@ -1250,7 +1250,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1398 +range-deletions-bytes-estimate: 1254 wait-pending-table-stats 000005 @@ -1259,7 +1259,7 @@ num-entries: 2 num-deletions: 2 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1398 +range-deletions-bytes-estimate: 1254 # Range deletions with varying overlap. @@ -1362,7 +1362,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 645 +range-deletions-bytes-estimate: 573 wait-pending-table-stats 000006 @@ -1371,7 +1371,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 634 +range-deletions-bytes-estimate: 562 wait-pending-table-stats 000004 @@ -1380,4 +1380,4 @@ num-entries: 2 num-deletions: 2 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 1279 +range-deletions-bytes-estimate: 1135 diff --git a/testdata/table_stats b/testdata/table_stats index 0a91af78f6..c172286fae 100644 --- a/testdata/table_stats +++ b/testdata/table_stats @@ -15,7 +15,7 @@ wait-pending-table-stats num-entries: 3 num-deletions: 1 num-range-key-sets: 0 -point-deletions-bytes-estimate: 64 +point-deletions-bytes-estimate: 57 range-deletions-bytes-estimate: 0 compact a-c @@ -181,7 +181,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 632 +range-deletions-bytes-estimate: 560 wait-pending-table-stats 000012 @@ -190,7 +190,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 632 +range-deletions-bytes-estimate: 560 # A table in L6 with two point keys blocks, each covered by distinct range dels. # The deletion estimate takes into account the contribution from both deleted @@ -346,7 +346,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 778 +range-deletions-bytes-estimate: 706 # Drop a range del and a range key del over the entire keyspace. This table can # delete everything underneath it. @@ -546,7 +546,7 @@ wait-pending-table-stats num-entries: 5 num-deletions: 2 num-range-key-sets: 0 -point-deletions-bytes-estimate: 112732 +point-deletions-bytes-estimate: 112572 range-deletions-bytes-estimate: 0 # Try a missized point tombstone. It should appear in the Metrics after the @@ -588,7 +588,7 @@ flush metadata-stats file=5 ---- -size: 726 +size: 654 # Just grab the physical sstable properties as these are used to construct the # virtual sstable properties. @@ -612,10 +612,7 @@ rocksdb: rocksdb.merge.operator: pebble.concatenate rocksdb.num.data.blocks: 1 rocksdb.merge.operands: 0 - rocksdb.prefix.extractor.name: nullptr - rocksdb.block.based.table.prefix.filtering: false rocksdb.property.collectors: [obsolete-key] - rocksdb.block.based.table.whole.key.filtering: false pebble: pebble.raw.point-tombstone.key.size: 1 rocksdb.comparator: pebble.internal.testkeys @@ -654,7 +651,7 @@ size: 53 properties file=7 ---- rocksdb.num.entries: 1 -rocksdb.raw.key.size: 2 +rocksdb.raw.key.size: 3 rocksdb.raw.value.size: 1 pebble.raw.point-tombstone.key.size: 1 rocksdb.deleted.keys: 1 @@ -662,7 +659,7 @@ rocksdb.deleted.keys: 1 properties file=8 ---- rocksdb.num.entries: 1 -rocksdb.raw.key.size: 2 +rocksdb.raw.key.size: 3 rocksdb.raw.value.size: 1 pebble.raw.point-tombstone.key.size: 1 rocksdb.deleted.keys: 1 @@ -673,7 +670,7 @@ wait-pending-table-stats num-entries: 1 num-deletions: 1 num-range-key-sets: 0 -point-deletions-bytes-estimate: 53 +point-deletions-bytes-estimate: 39 range-deletions-bytes-estimate: 0 wait-pending-table-stats @@ -682,7 +679,7 @@ wait-pending-table-stats num-entries: 1 num-deletions: 1 num-range-key-sets: 0 -point-deletions-bytes-estimate: 53 +point-deletions-bytes-estimate: 39 range-deletions-bytes-estimate: 0 # Create an sstable with a range key set. @@ -723,10 +720,7 @@ rocksdb: rocksdb.merge.operator: pebble.concatenate rocksdb.num.data.blocks: 1 rocksdb.merge.operands: 0 - rocksdb.prefix.extractor.name: nullptr - rocksdb.block.based.table.prefix.filtering: false rocksdb.property.collectors: [obsolete-key] - rocksdb.block.based.table.whole.key.filtering: false pebble: pebble.num.range-key-dels: 0 pebble.num.range-key-sets: 1 @@ -738,7 +732,7 @@ pebble: metadata-stats file=10 ---- -size: 828 +size: 756 build ext2 set z z @@ -848,10 +842,7 @@ rocksdb: rocksdb.merge.operator: pebble.concatenate rocksdb.num.data.blocks: 1 rocksdb.merge.operands: 0 - rocksdb.prefix.extractor.name: nullptr - rocksdb.block.based.table.prefix.filtering: false rocksdb.property.collectors: [obsolete-key] - rocksdb.block.based.table.whole.key.filtering: false pebble: rocksdb.comparator: pebble.internal.testkeys rocksdb.merge.operator: pebble.concatenate @@ -899,7 +890,7 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 661 +range-deletions-bytes-estimate: 589 wait-pending-table-stats 000021 @@ -908,4 +899,4 @@ num-entries: 1 num-deletions: 1 num-range-key-sets: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 660 +range-deletions-bytes-estimate: 588 diff --git a/tool/sstable.go b/tool/sstable.go index fba651c9b2..a71a5a9a46 100644 --- a/tool/sstable.go +++ b/tool/sstable.go @@ -338,8 +338,6 @@ func (s *sstableT) runProperties(cmd *cobra.Command, args []string) { fmt.Fprintf(tw, "comparer\t%s\n", r.Properties.ComparerName) fmt.Fprintf(tw, "merger\t%s\n", formatNull(r.Properties.MergerName)) fmt.Fprintf(tw, "filter\t%s\n", formatNull(r.Properties.FilterPolicyName)) - fmt.Fprintf(tw, " prefix\t%t\n", r.Properties.PrefixFiltering) - fmt.Fprintf(tw, " whole-key\t%t\n", r.Properties.WholeKeyFiltering) fmt.Fprintf(tw, "compression\t%s\n", r.Properties.CompressionName) fmt.Fprintf(tw, " options\t%s\n", r.Properties.CompressionOptions) fmt.Fprintf(tw, "user properties\t\n") diff --git a/tool/testdata/sstable_layout b/tool/testdata/sstable_layout index 90373aea63..4c686d329c 100644 --- a/tool/testdata/sstable_layout +++ b/tool/testdata/sstable_layout @@ -22,10 +22,10 @@ h.sst 13752 data (156) 13913 index (245) 14163 range-del (421) - 14589 properties (513) - 15107 meta-index (61) - 15173 footer (53) - 15226 EOF + 14589 properties (441) + 15035 meta-index (61) + 15101 footer (53) + 15154 EOF sstable layout ../sstable/testdata/h.table-bloom.no-compression.sst @@ -48,10 +48,10 @@ h.table-bloom.no-compression.sst 26799 filter (2245) 29049 index (325) 29379 range-del (421) - 29805 properties (557) - 30367 meta-index (112) - 30484 footer (53) - 30537 EOF + 29805 properties (485) + 30295 meta-index (112) + 30412 footer (53) + 30465 EOF sstable layout ../sstable/testdata/h.no-compression.two_level_index.sst @@ -76,10 +76,10 @@ h.no-compression.two_level_index.sst 27047 index (95) 27147 top-index (70) 27222 range-del (421) - 27648 properties (559) - 28212 meta-index (63) - 28280 footer (53) - 28333 EOF + 27648 properties (487) + 28140 meta-index (63) + 28208 footer (53) + 28261 EOF sstable layout -v @@ -3744,45 +3744,42 @@ h.no-compression.two_level_index.sst 27631 [restart 27523] 27635 [restart 27546] 27643 [trailer compression=none checksum=0xb93b31c5] - 27648 properties (559) + 27648 properties (487) 27648 rocksdb.block.based.table.index.type (43) [restart] - 27691 rocksdb.block.based.table.prefix.filtering (20) - 27711 rocksdb.block.based.table.whole.key.filtering (23) - 27734 rocksdb.comparator (39) - 27773 rocksdb.compression (23) - 27796 rocksdb.compression_options (106) - 27902 rocksdb.data.size (15) - 27917 rocksdb.deleted.keys (15) - 27932 rocksdb.external_sst_file.version (32) - 27964 rocksdb.filter.size (15) - 27979 rocksdb.index.partitions (20) - 27999 rocksdb.index.size (9) - 28008 rocksdb.merge.operands (18) - 28026 rocksdb.merge.operator (13) - 28039 rocksdb.num.data.blocks (19) - 28058 rocksdb.num.entries (12) - 28070 rocksdb.num.range-deletions (19) - 28089 rocksdb.prefix.extractor.name (31) - 28120 rocksdb.property.collectors (22) - 28142 rocksdb.raw.key.size (18) - 28160 rocksdb.raw.value.size (15) - 28175 rocksdb.top-level.index.size (24) - 28199 [restart 27648] - 28207 [trailer compression=none checksum=0x2d96f92a] - 28212 meta-index (63) - 28212 rocksdb.properties block:27648/559 [restart] - 28238 rocksdb.range_del block:27222/421 [restart] - 28263 [restart 28212] - 28267 [restart 28238] - 28275 [trailer compression=none checksum=0xf7259c47] - 28280 footer (53) - 28280 checksum type: crc32c - 28281 meta: offset=28212, length=63 - 28285 index: offset=27147, length=70 - 28289 [padding] - 28321 version: 1 - 28325 magic number: 0xf09faab3f09faab3 - 28333 EOF + 27691 rocksdb.comparator (39) + 27730 rocksdb.compression (23) + 27753 rocksdb.compression_options (106) + 27859 rocksdb.data.size (15) + 27874 rocksdb.deleted.keys (15) + 27889 rocksdb.external_sst_file.version (32) + 27921 rocksdb.filter.size (15) + 27936 rocksdb.index.partitions (20) + 27956 rocksdb.index.size (9) + 27965 rocksdb.merge.operands (18) + 27983 rocksdb.merge.operator (13) + 27996 rocksdb.num.data.blocks (19) + 28015 rocksdb.num.entries (12) + 28027 rocksdb.num.range-deletions (19) + 28046 rocksdb.property.collectors (24) + 28070 rocksdb.raw.key.size (18) + 28088 rocksdb.raw.value.size (15) + 28103 rocksdb.top-level.index.size (24) + 28127 [restart 27648] + 28135 [trailer compression=none checksum=0x789e68f8] + 28140 meta-index (63) + 28140 rocksdb.properties block:27648/487 [restart] + 28166 rocksdb.range_del block:27222/421 [restart] + 28191 [restart 28140] + 28195 [restart 28166] + 28203 [trailer compression=none checksum=0xb36870b3] + 28208 footer (53) + 28208 checksum type: crc32c + 28209 meta: offset=28140, length=63 + 28213 index: offset=27147, length=70 + 28217 [padding] + 28249 version: 1 + 28253 magic number: 0xf09faab3f09faab3 + 28261 EOF sstable layout -v diff --git a/tool/testdata/sstable_properties b/tool/testdata/sstable_properties index 810db1311e..593e93fe94 100644 Binary files a/tool/testdata/sstable_properties and b/tool/testdata/sstable_properties differ