storage: set L0CompactionConcurrency to 2

The default in Pebble, 10, delays increasing the compaction
concurrency which leads to the bad behavior discussed in
cockroachdb/pebble#2832 (comment).
The value of 10 was chosen when admission control was not shaping
incoming traffic until there were 20 sub-levels. Admission control now
shapes regular traffic starting at 5 sub-levels and elastic traffic
starting at 1 sub-level.

Informs #104862
Informs cockroachdb/pebble#2832

Epic: none

Release note: None

pkg/storage/mvcc.go

@@ -129,6 +129,19 @@ var rocksdbConcurrency = envutil.EnvOrDefaultInt(
 		return max
 	}())
 
+// The sub-level threshold at which to allow an increase in compaction
+// concurrency. The maximum is still controlled by
+// pebble.Options.MaxConcurrentCompactions. The default of 2 will allow an
+// additional compaction (so total 1 + 1 = 2 compactions) when the sub-level
+// count is 2, and increment concurrency by 1 whenever sub-level count
+// increases by 2 (so 1 + 3 = 4 compactions) when sub-level count is 6.
+// Admission control starts shaping regular traffic at a sub-level count of 5,
+// and elastic traffic at a sub-level count of 1, hence this default of 2. See
+// https://github.com/cockroachdb/pebble/issues/2832#issuecomment-1699743392
+// for some discussion on the bad behavior caused by not setting this option.
+var l0SubLevelCompactionConcurrency = envutil.EnvOrDefaultInt(
+	"COCKROACH_L0_SUB_LEVEL_CONCURRENCY", 2)
+
 // MakeValue returns the inline value.
 func MakeValue(meta enginepb.MVCCMetadata) roachpb.Value {
 	return roachpb.Value{RawBytes: meta.RawBytes}

pkg/storage/pebble.go

@@ -571,8 +571,9 @@ func DefaultPebbleOptions() *pebble.Options {
 	}
 
 	opts := &pebble.Options{
-		Comparer:                    EngineComparer,
-		FS:                          vfs.Default,
+		Comparer: EngineComparer,
+		FS:       vfs.Default,
+		// A value of 2 triggers a compaction when there is 1 sub-level.
 		L0CompactionThreshold:       2,
 		L0StopWritesThreshold:       1000,
 		LBaseMaxBytes:               64 << 20, // 64 MB
@@ -584,6 +585,7 @@ func DefaultPebbleOptions() *pebble.Options {
 		BlockPropertyCollectors:     PebbleBlockPropertyCollectors,
 		FormatMajorVersion:          MinimumSupportedFormatVersion,
 	}
+	opts.Experimental.L0CompactionConcurrency = l0SubLevelCompactionConcurrency
 	// Automatically flush 10s after the first range tombstone is added to a
 	// memtable. This ensures that we can reclaim space even when there's no
 	// activity on the database generating flushes.