db: add range key count to table stats

Range key-related statistics are not currently calculated and stored in a table's `FileMetadata`, and are required for compaction picking. Add a `NumRangeKeys` field to `manifest.TableStats` which is populated by the total number of range keys present in a table (the sum of range key sets, unsets and deletes).
cockroachdb · Apr 4, 2022 · c8e7d41 · c8e7d41
1 parent ec4dcaa
commit c8e7d41
Show file tree

Hide file tree

Showing 11 changed files with 82 additions and 2 deletions.
diff --git a/data_test.go b/data_test.go
@@ -796,6 +796,7 @@ func runTableStatsCmd(td *datadriven.TestData, d *DB) string {
 			var b bytes.Buffer
 			fmt.Fprintf(&b, "num-entries: %d\n", f.Stats.NumEntries)
 			fmt.Fprintf(&b, "num-deletions: %d\n", f.Stats.NumDeletions)
+			fmt.Fprintf(&b, "num-range-keys: %d\n", f.Stats.NumRangeKeys)
 			fmt.Fprintf(&b, "point-deletions-bytes-estimate: %d\n", f.Stats.PointDeletionsBytesEstimate)
 			fmt.Fprintf(&b, "range-deletions-bytes-estimate: %d\n", f.Stats.RangeDeletionsBytesEstimate)
 			return b.String()

diff --git a/internal/manifest/version.go b/internal/manifest/version.go
@@ -50,6 +50,8 @@ type TableStats struct {
 	NumEntries uint64
 	// The number of point and range deletion entries in the table.
 	NumDeletions uint64
+	// NumRangeKeys is the total number of range keys in the table.
+	NumRangeKeys uint64
 	// Estimate of the total disk space that may be dropped by this table's
 	// point deletions by compacting them.
 	PointDeletionsBytesEstimate uint64

diff --git a/table_stats.go b/table_stats.go
@@ -262,6 +262,10 @@ func (d *DB) loadTableStats(
 				return
 			}
 		}
+		// TODO(travers): Once we have real-world data, consider collecting
+		// additional stats that may provide improved heuristics for compaction
+		// picking.
+		stats.NumRangeKeys = r.Properties.NumRangeKeys()
 		return
 	})
 	if err != nil {
@@ -532,6 +536,7 @@ func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties)
 		Valid:                       true,
 		NumEntries:                  props.NumEntries,
 		NumDeletions:                props.NumDeletions,
+		NumRangeKeys:                props.NumRangeKeys(),
 		PointDeletionsBytesEstimate: pointEstimate,
 		RangeDeletionsBytesEstimate: 0,
 	}

diff --git a/table_stats_test.go b/table_stats_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/datadriven"
 	"github.com/cockroachdb/pebble/internal/keyspan"
+	"github.com/cockroachdb/pebble/internal/testkeys"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/stretchr/testify/require"
 )
@@ -28,6 +29,9 @@ func TestTableStats(t *testing.T) {
 		},
 	}
 	opts.DisableAutomaticCompactions = true
+	opts.Comparer = testkeys.Comparer
+	opts.Experimental.RangeKeys = new(RangeKeysArena)
+	opts.FormatMajorVersion = FormatRangeKeys
 
 	d, err := Open("", opts)
 	require.NoError(t, err)
@@ -92,6 +96,18 @@ func TestTableStats(t *testing.T) {
 			d.mu.Unlock()
 			return s
 
+		case "ingest":
+			if err = runBuildCmd(td, d, d.opts.FS); err != nil {
+				return err.Error()
+			}
+			if err = runIngestCmd(td, d, d.opts.FS); err != nil {
+				return err.Error()
+			}
+			d.mu.Lock()
+			s := d.mu.versions.currentVersion().String()
+			d.mu.Unlock()
+			return s
+
 		case "wait-pending-table-stats":
 			return runTableStatsCmd(td, d)
 

diff --git a/testdata/compaction_delete_only_hints b/testdata/compaction_delete_only_hints
@@ -206,6 +206,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 26
 

diff --git a/testdata/compaction_tombstones b/testdata/compaction_tombstones
@@ -13,6 +13,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 2
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
 
@@ -34,6 +35,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 2
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
 
@@ -54,6 +56,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 26
 
@@ -76,6 +79,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
 
@@ -114,6 +118,7 @@ wait-pending-table-stats
 ----
 num-entries: 6
 num-deletions: 2
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 76
 
@@ -146,6 +151,7 @@ wait-pending-table-stats
 ----
 num-entries: 11
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 149
 range-deletions-bytes-estimate: 0
 
@@ -189,6 +195,7 @@ wait-pending-table-stats
 ----
 num-entries: 5
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 16488
 
@@ -225,6 +232,7 @@ wait-pending-table-stats
 ----
 num-entries: 3
 num-deletions: 3
+num-range-keys: 0
 point-deletions-bytes-estimate: 13167
 range-deletions-bytes-estimate: 0
 

diff --git a/testdata/ingest b/testdata/ingest
@@ -74,6 +74,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 0
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
 
@@ -346,6 +347,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 2
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1666
 

diff --git a/testdata/manual_compaction b/testdata/manual_compaction
@@ -86,6 +86,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1552
 
@@ -103,6 +104,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 776
 

diff --git a/testdata/manual_compaction_set_with_del b/testdata/manual_compaction_set_with_del
@@ -86,6 +86,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1552
 
@@ -103,6 +104,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 776
 

diff --git a/testdata/range_del b/testdata/range_del
@@ -1390,6 +1390,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
 
@@ -1398,6 +1399,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 836
 
@@ -1406,6 +1408,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1672
 
@@ -1414,6 +1417,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 2
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1672
 
@@ -1453,6 +1457,7 @@ wait-pending-table-stats
 ----
 num-entries: 4
 num-deletions: 0
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
 
@@ -1461,6 +1466,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 42
 
@@ -1469,6 +1475,7 @@ wait-pending-table-stats
 ----
 num-entries: 3
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 68
 
@@ -1477,6 +1484,7 @@ wait-pending-table-stats
 ----
 num-entries: 4
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 100
 
@@ -1512,6 +1520,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 782
 
@@ -1520,6 +1529,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 771
 
@@ -1528,5 +1538,6 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 2
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1553
diff --git a/testdata/table_stats b/testdata/table_stats
@@ -14,6 +14,7 @@ wait-pending-table-stats
 ----
 num-entries: 3
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
 
@@ -38,6 +39,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 51
 
@@ -56,6 +58,7 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 51
 
@@ -93,6 +96,7 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 0
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
 
@@ -175,16 +179,18 @@ wait-pending-table-stats
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
-range-deletions-bytes-estimate: 771
+range-deletions-bytes-estimate: 769
 
 wait-pending-table-stats
 000012
 ----
 num-entries: 1
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
-range-deletions-bytes-estimate: 771
+range-deletions-bytes-estimate: 769
 
 define snapshots=(10)
 L6
@@ -198,5 +204,29 @@ wait-pending-table-stats
 ----
 num-entries: 2
 num-deletions: 1
+num-range-keys: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 26
+
+# Ingest a table with range keys.
+# TODO(travers): Once range keys in batches are flushed to tables, this testcase
+# can be updated to use the batch / flush combination.
+
+ingest ext0
+range-key-set a b @1 foo
+range-key-unset a b @2
+range-key-del a b
+----
+5:
+  000005:[a#1,RANGEKEYSET-b#72057594037927935,RANGEKEYDEL]
+6:
+  000004:[a#15,RANGEDEL-z#72057594037927935,RANGEDEL]
+
+wait-pending-table-stats
+000005
+----
+num-entries: 0
+num-deletions: 0
+num-range-keys: 3
+point-deletions-bytes-estimate: 0
+range-deletions-bytes-estimate: 0