diff --git a/data_test.go b/data_test.go index 10c5b19490..89fabaf7a0 100644 --- a/data_test.go +++ b/data_test.go @@ -796,6 +796,7 @@ func runTableStatsCmd(td *datadriven.TestData, d *DB) string { var b bytes.Buffer fmt.Fprintf(&b, "num-entries: %d\n", f.Stats.NumEntries) fmt.Fprintf(&b, "num-deletions: %d\n", f.Stats.NumDeletions) + fmt.Fprintf(&b, "num-range-keys: %d\n", f.Stats.NumRangeKeys) fmt.Fprintf(&b, "point-deletions-bytes-estimate: %d\n", f.Stats.PointDeletionsBytesEstimate) fmt.Fprintf(&b, "range-deletions-bytes-estimate: %d\n", f.Stats.RangeDeletionsBytesEstimate) return b.String() diff --git a/internal/manifest/version.go b/internal/manifest/version.go index a4be804a02..aefb321e1c 100644 --- a/internal/manifest/version.go +++ b/internal/manifest/version.go @@ -50,6 +50,8 @@ type TableStats struct { NumEntries uint64 // The number of point and range deletion entries in the table. NumDeletions uint64 + // NumRangeKeys is the total number of range keys in the table. + NumRangeKeys uint64 // Estimate of the total disk space that may be dropped by this table's // point deletions by compacting them. PointDeletionsBytesEstimate uint64 diff --git a/table_stats.go b/table_stats.go index 3a98bf2c9f..e127919fc5 100644 --- a/table_stats.go +++ b/table_stats.go @@ -262,6 +262,10 @@ func (d *DB) loadTableStats( return } } + // TODO(travers): Once we have real-world data, consider collecting + // additional stats that may provide improved heuristics for compaction + // picking. + stats.NumRangeKeys = r.Properties.NumRangeKeys() return }) if err != nil { @@ -532,6 +536,7 @@ func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) Valid: true, NumEntries: props.NumEntries, NumDeletions: props.NumDeletions, + NumRangeKeys: props.NumRangeKeys(), PointDeletionsBytesEstimate: pointEstimate, RangeDeletionsBytesEstimate: 0, } diff --git a/table_stats_test.go b/table_stats_test.go index a215ad44d5..4c79608ee8 100644 --- a/table_stats_test.go +++ b/table_stats_test.go @@ -11,6 +11,7 @@ import ( "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/datadriven" "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/testkeys" "github.com/cockroachdb/pebble/vfs" "github.com/stretchr/testify/require" ) @@ -28,6 +29,9 @@ func TestTableStats(t *testing.T) { }, } opts.DisableAutomaticCompactions = true + opts.Comparer = testkeys.Comparer + opts.Experimental.RangeKeys = new(RangeKeysArena) + opts.FormatMajorVersion = FormatRangeKeys d, err := Open("", opts) require.NoError(t, err) @@ -92,6 +96,18 @@ func TestTableStats(t *testing.T) { d.mu.Unlock() return s + case "ingest": + if err = runBuildCmd(td, d, d.opts.FS); err != nil { + return err.Error() + } + if err = runIngestCmd(td, d, d.opts.FS); err != nil { + return err.Error() + } + d.mu.Lock() + s := d.mu.versions.currentVersion().String() + d.mu.Unlock() + return s + case "wait-pending-table-stats": return runTableStatsCmd(td, d) diff --git a/testdata/compaction_delete_only_hints b/testdata/compaction_delete_only_hints index 245edcd6dc..871c6c955f 100644 --- a/testdata/compaction_delete_only_hints +++ b/testdata/compaction_delete_only_hints @@ -206,6 +206,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 26 diff --git a/testdata/compaction_tombstones b/testdata/compaction_tombstones index 5b0aa37597..b1cd72482d 100644 --- a/testdata/compaction_tombstones +++ b/testdata/compaction_tombstones @@ -13,6 +13,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 2 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 0 @@ -34,6 +35,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 2 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 0 @@ -54,6 +56,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 26 @@ -76,6 +79,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 0 @@ -114,6 +118,7 @@ wait-pending-table-stats ---- num-entries: 6 num-deletions: 2 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 76 @@ -146,6 +151,7 @@ wait-pending-table-stats ---- num-entries: 11 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 149 range-deletions-bytes-estimate: 0 @@ -189,6 +195,7 @@ wait-pending-table-stats ---- num-entries: 5 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 16488 @@ -225,6 +232,7 @@ wait-pending-table-stats ---- num-entries: 3 num-deletions: 3 +num-range-keys: 0 point-deletions-bytes-estimate: 13167 range-deletions-bytes-estimate: 0 diff --git a/testdata/ingest b/testdata/ingest index 89fd2133c4..65408898e6 100644 --- a/testdata/ingest +++ b/testdata/ingest @@ -74,6 +74,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 0 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 0 @@ -346,6 +347,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 2 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 1666 diff --git a/testdata/manual_compaction b/testdata/manual_compaction index 21439c377b..78c431387e 100644 --- a/testdata/manual_compaction +++ b/testdata/manual_compaction @@ -86,6 +86,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 1552 @@ -103,6 +104,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 776 diff --git a/testdata/manual_compaction_set_with_del b/testdata/manual_compaction_set_with_del index c414c4a5fc..c361ebcf93 100644 --- a/testdata/manual_compaction_set_with_del +++ b/testdata/manual_compaction_set_with_del @@ -86,6 +86,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 1552 @@ -103,6 +104,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 776 diff --git a/testdata/range_del b/testdata/range_del index bbfa454921..ff69063407 100644 --- a/testdata/range_del +++ b/testdata/range_del @@ -1390,6 +1390,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 0 @@ -1398,6 +1399,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 836 @@ -1406,6 +1408,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 1672 @@ -1414,6 +1417,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 2 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 1672 @@ -1453,6 +1457,7 @@ wait-pending-table-stats ---- num-entries: 4 num-deletions: 0 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 0 @@ -1461,6 +1466,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 42 @@ -1469,6 +1475,7 @@ wait-pending-table-stats ---- num-entries: 3 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 68 @@ -1477,6 +1484,7 @@ wait-pending-table-stats ---- num-entries: 4 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 100 @@ -1512,6 +1520,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 782 @@ -1520,6 +1529,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 771 @@ -1528,5 +1538,6 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 2 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 1553 diff --git a/testdata/table_stats b/testdata/table_stats index 173661cd5b..1d0e8f4c4a 100644 --- a/testdata/table_stats +++ b/testdata/table_stats @@ -14,6 +14,7 @@ wait-pending-table-stats ---- num-entries: 3 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 0 @@ -38,6 +39,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 51 @@ -56,6 +58,7 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 51 @@ -93,6 +96,7 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 0 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 0 @@ -175,16 +179,18 @@ wait-pending-table-stats ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 771 +range-deletions-bytes-estimate: 769 wait-pending-table-stats 000012 ---- num-entries: 1 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 -range-deletions-bytes-estimate: 771 +range-deletions-bytes-estimate: 769 define snapshots=(10) L6 @@ -198,5 +204,29 @@ wait-pending-table-stats ---- num-entries: 2 num-deletions: 1 +num-range-keys: 0 point-deletions-bytes-estimate: 0 range-deletions-bytes-estimate: 26 + +# Ingest a table with range keys. +# TODO(travers): Once range keys in batches are flushed to tables, this testcase +# can be updated to use the batch / flush combination. + +ingest ext0 +range-key-set a b @1 foo +range-key-unset a b @2 +range-key-del a b +---- +5: + 000005:[a#1,RANGEKEYSET-b#72057594037927935,RANGEKEYDEL] +6: + 000004:[a#15,RANGEDEL-z#72057594037927935,RANGEDEL] + +wait-pending-table-stats +000005 +---- +num-entries: 0 +num-deletions: 0 +num-range-keys: 3 +point-deletions-bytes-estimate: 0 +range-deletions-bytes-estimate: 0