clusterversion,storage: add explicit version for split user keys

Pebble ceased splitting user keys across multiple sstables in a single
level in cockroachdb/pebble@a860bbad. A Pebble format major version was
added in 22.1 to mark existing tables with split user that formed an
"atomic compaction unit" for compaction. The compaction to recombine the
split keys happens at a low priority within Pebble, so there is no
guarantee that these tables have been recombined.

Pebble range key support depends on having split user keys recombined.
As such, the Pebble migration to upgrade to a version that support range
keys must first perform a blocking migration to compact tables with
split user keys.

Currently, the Cockroach cluster version
`EnsurePebbleFormatVersionRangeKeys` ratchets the Pebble version through
both the split user keys migration, up to the version that supports
range keys. Strictly speaking, the Pebble format version migration will
take care of the sequencing.

However, to provide better visibility into the various Pebble
migrations, and to cater for the fact that the split user keys migration
may take some time to recombine the necessary tables, a dedicated
cluster version for the split keys migration is added. This dedicated
cluster version makes it unambiguous what's blocking the Cockroach
version finalization.

Release note (backward-incompatible change, ops change): A cluster
version is added to allow Pebble to recombine certain SSTables
(specifically, user keys that are split across multiple files in a level
of the LSM). Recombining the split user keys is required for supporting
the range keys feature.  The migration to recombine the SSTables is
expected to be short (split user keys are rare in practice), but will
block subsequent migrations until all tables have been recombined. The
`storage.marked-for-compaction-files` time series metric can show the
progress of the migration.

Close #84012.

docs/generated/settings/settings-for-tenants.txt

@@ -283,4 +283,4 @@ trace.jaeger.agent	string		the address of a Jaeger agent to receive traces using
 trace.opentelemetry.collector	string		address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.
 trace.span_registry.enabled	boolean	true	if set, ongoing traces can be seen at https://<ui>/#/debug/tracez
 trace.zipkin.collector	string		the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.
-version	version	22.1-28	set the active cluster version in the format '<major>.<minor>'
+version	version	22.1-30	set the active cluster version in the format '<major>.<minor>'

docs/generated/settings/settings.html

@@ -214,6 +214,6 @@
 <tr><td><code>trace.opentelemetry.collector</code></td><td>string</td><td><code></code></td><td>address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.</td></tr>
 <tr><td><code>trace.span_registry.enabled</code></td><td>boolean</td><td><code>true</code></td><td>if set, ongoing traces can be seen at https://<ui>/#/debug/tracez</td></tr>
 <tr><td><code>trace.zipkin.collector</code></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.</td></tr>
-<tr><td><code>version</code></td><td>version</td><td><code>22.1-28</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
+<tr><td><code>version</code></td><td>version</td><td><code>22.1-30</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
 </tbody>
 </table>

pkg/clusterversion/cockroach_versions.go

@@ -289,6 +289,10 @@ const (
 
 	// LocalTimestamps enables the use of local timestamps in MVCC values.
 	LocalTimestamps
+	// PebbleFormatSplitUserKeysMarkedCompacted updates the Pebble format
+	// version that recombines all user keys that may be split across multiple
+	// files into a single table.
+	PebbleFormatSplitUserKeysMarkedCompacted
 	// EnsurePebbleFormatVersionRangeKeys is the first step of a two-part
 	// migration that bumps Pebble's format major version to a version that
 	// supports range keys.
@@ -517,53 +521,57 @@ var versionsSingleton = keyedVersions{
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 4},
 	},
 	{
-		Key:     EnsurePebbleFormatVersionRangeKeys,
+		Key:     PebbleFormatSplitUserKeysMarkedCompacted,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 6},
 	},
 	{
-		Key:     EnablePebbleFormatVersionRangeKeys,
+		Key:     EnsurePebbleFormatVersionRangeKeys,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 8},
 	},
 	{
-		Key:     TrigramInvertedIndexes,
+		Key:     EnablePebbleFormatVersionRangeKeys,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 10},
 	},
 	{
-		Key:     RemoveGrantPrivilege,
+		Key:     TrigramInvertedIndexes,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 12},
 	},
 	{
-		Key:     MVCCRangeTombstones,
+		Key:     RemoveGrantPrivilege,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 14},
 	},
 	{
-		Key:     UpgradeSequenceToBeReferencedByID,
+		Key:     MVCCRangeTombstones,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 16},
 	},
 	{
-		Key:     SampledStmtDiagReqs,
+		Key:     UpgradeSequenceToBeReferencedByID,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 18},
 	},
 	{
-		Key:     AddSSTableTombstones,
+		Key:     SampledStmtDiagReqs,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 20},
 	},
 	{
-		Key:     SystemPrivilegesTable,
+		Key:     AddSSTableTombstones,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 22},
 	},
 	{
-		Key:     EnablePredicateProjectionChangefeed,
+		Key:     SystemPrivilegesTable,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 24},
 	},
 	{
-		Key:     AlterSystemSQLInstancesAddLocality,
+		Key:     EnablePredicateProjectionChangefeed,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 26},
 	},
 	{
-		Key:     SystemExternalConnectionsTable,
+		Key:     AlterSystemSQLInstancesAddLocality,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 28},
 	},
+	{
+		Key:     SystemExternalConnectionsTable,
+		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 30},
+	},
 
 	// *************************************************
 	// Step (2): Add new versions here.

pkg/storage/pebble.go

@@ -1810,6 +1810,10 @@ func (p *Pebble) SetMinVersion(version roachpb.Version) error {
 		if formatVers < pebble.FormatRangeKeys {
 			formatVers = pebble.FormatRangeKeys
 		}
+	case !version.Less(clusterversion.ByKey(clusterversion.PebbleFormatSplitUserKeysMarkedCompacted)):
+		if formatVers < pebble.FormatMarkedCompacted {
+			formatVers = pebble.FormatMarkedCompacted
+		}
 	case !version.Less(clusterversion.ByKey(clusterversion.PebbleFormatSplitUserKeysMarked)):
 		if formatVers < pebble.FormatSplitUserKeysMarked {
 			formatVers = pebble.FormatSplitUserKeysMarked