catalog: add SystemDatabaseSchemaVersion field to DatabaseDescriptor

pkg/sql/catalog/bootstrap/BUILD.bazel

@@ -52,6 +52,7 @@ go_test(
         "//pkg/security/securityassets",
         "//pkg/security/securitytest",
         "//pkg/server",
+        "//pkg/sql/catalog/systemschema",
         "//pkg/testutils",
         "//pkg/testutils/datapathutils",
         "//pkg/testutils/serverutils",

pkg/sql/catalog/bootstrap/bootstrap_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/config/zonepb"
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema"
 	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -80,24 +81,18 @@ func TestInitialValuesToString(t *testing.T) {
 
 	datadriven.Walk(t, datapathutils.TestDataPath(t), func(t *testing.T, path string) {
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
-			codec := keys.SystemSQLCodec
+			var tenantID uint64
 			switch d.Cmd {
 			case "system":
-				break
-
 			case "tenant":
-				const dummyTenantID = 12345
-				codec = keys.MakeSQLCodec(roachpb.MustMakeTenantID(dummyTenantID))
-
+				tenantID = 12345
 			default:
 				t.Fatalf("unexpected command %q", d.Cmd)
 			}
 			var expectedHash string
 			d.ScanArgs(t, "hash", &expectedHash)
-			ms := MakeMetadataSchema(codec, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef())
-			result := InitialValuesToString(ms)
-			h := sha256.Sum256([]byte(result))
-			if actualHash := hex.EncodeToString(h[:]); expectedHash != actualHash {
+			initialValues, actualHash := getAndHashInitialValuesToString(tenantID)
+			if expectedHash != actualHash {
 				t.Errorf(`Unexpected hash value %s for %s.
 If you're seeing this error message, this means that the bootstrapped system
 schema has changed. Assuming that this is expected:
@@ -109,11 +104,20 @@ schema has changed. Assuming that this is expected:
   hardcoded literals in the main development branch as well as any subsequent
   release branches that need to be updated also.`, actualHash, d.Cmd)
 			}
-			return result
+
+			return initialValues
 		})
 	})
 }
 
+func getAndHashInitialValuesToString(tenantID uint64) (initialValues string, hash string) {
+	ms := makeMetadataSchema(tenantID)
+	initialValues = InitialValuesToString(ms)
+	h := sha256.Sum256([]byte(initialValues))
+	hash = hex.EncodeToString(h[:])
+	return initialValues, hash
+}
+
 func TestRoundTripInitialValuesStringRepresentation(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
@@ -158,3 +162,30 @@ func makeMetadataSchema(tenantID uint64) MetadataSchema {
 	}
 	return MakeMetadataSchema(codec, zonepb.DefaultZoneConfigRef(), zonepb.DefaultSystemZoneConfigRef())
 }
+
+// TestSystemDatabaseSchemaBootstrapVersionBumped serves as a reminder to bump
+// systemschema.SystemDatabaseSchemaBootstrapVersion whenever a new upgrade
+// creates or modifies the schema of system tables. We unfortunately cannot
+// programmatically determine if an upgrade should bump the version so by
+// adding a test failure when the initial values change, the programmer and
+// code reviewers are reminded to manually check whether the version should
+// be bumped.
+func TestSystemDatabaseSchemaBootstrapVersionBumped(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	// If you need to update this value (i.e. failed this test), check whether
+	// you need to bump systemschema.SystemDatabaseSchemaBootstrapVersion too.
+	const prevSystemHash = "b23cc2e4f5d8e1a3d3bdd514addb3bd9af3eff65cd485fa2822ced1a6a2e8fd4"
+	_, curSystemHash := getAndHashInitialValuesToString(0 /* tenantID */)
+
+	if prevSystemHash != curSystemHash {
+		t.Fatalf(
+			`Check whether you need to bump systemschema.SystemDatabaseSchemaBootstrapVersion
+and then update prevSystemHash to %q.
+The current value of SystemDatabaseSchemaBootstrapVersion is %s.`,
+			curSystemHash,
+			systemschema.SystemDatabaseSchemaBootstrapVersion,
+		)
+	}
+}

pkg/sql/catalog/bootstrap/testdata/testdata

@@ -1,8 +1,8 @@
-system hash=a598cb92096a1b13ecc8fe8f0d6f70a0812a05617a84c0061a09d6393f50d97b
+system hash=b23cc2e4f5d8e1a3d3bdd514addb3bd9af3eff65cd485fa2822ced1a6a2e8fd4
 ----
 [{"key":"04646573632d696467656e","value":"01c801"}
 ,{"key":"8b"}
-,{"key":"8b89898a89","value":"0312390a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518022200280140004a00"}
+,{"key":"8b89898a89","value":"0312450a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518022200280140004a006a0a08d7843d100118002014"}
 ,{"key":"8b898b8a89","value":"030a88030a0a64657363726970746f721803200128013a0042270a02696410011a0c08011040180030005014600020003000680070007800800100880100980100422f0a0a64657363726970746f7210021a0c08081000180030005011600020013000680070007800800100880100980100480352710a077072696d61727910011801220269642a0a64657363726970746f72300140004a10080010001a00200028003000380040005a0070027a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060026a210a0b0a0561646d696e102018200a0a0a04726f6f741020182012046e6f64651802800101880103980100b201130a077072696d61727910001a02696420012800b201240a1066616d5f325f64657363726970746f7210021a0a64657363726970746f7220022802b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300d00300"}
 ,{"key":"8b898c8a89","value":"030ac1050a0575736572731804200128013a00422d0a08757365726e616d6510011a0c0807100018003000501960002000300068007000780080010088010098010042330a0e68617368656450617373776f726410021a0c0808100018003000501160002001300068007000780080010088010098010042320a066973526f6c6510031a0c08001000180030005010600020002a0566616c73653000680070007800800100880100980100422c0a07757365725f696410041a0c080c100018003000501a60002000300068007000780080010088010098010048055290010a077072696d617279100118012208757365726e616d652a0e68617368656450617373776f72642a066973526f6c652a07757365725f6964300140004a10080010001a00200028003000380040005a007002700370047a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00102e00100e90100000000000000005a740a1175736572735f757365725f69645f696478100218012207757365725f69643004380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060036a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201240a077072696d61727910001a08757365726e616d651a07757365725f6964200120042804b2012c0a1466616d5f325f68617368656450617373776f726410021a0e68617368656450617373776f726420022802b2011c0a0c66616d5f335f6973526f6c6510031a066973526f6c6520032803b80104c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880303a80300b00300d00300"}
 ,{"key":"8b898d8a89","value":"030af7020a057a6f6e65731805200128013a0042270a02696410011a0c08011040180030005014600020003000680070007800800100880100980100422b0a06636f6e66696710021a0c080810001800300050116000200130006800700078008001008801009801004803526d0a077072696d61727910011801220269642a06636f6e666967300140004a10080010001a00200028003000380040005a0070027a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201130a077072696d61727910001a02696420012800b2011c0a0c66616d5f325f636f6e66696710021a06636f6e66696720022802b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300d00300"}
@@ -182,10 +182,10 @@ system hash=a598cb92096a1b13ecc8fe8f0d6f70a0812a05617a84c0061a09d6393f50d97b
 ,{"key":"c6898888","value":"0102"}
 ]
 
-tenant hash=2c06a652d163a10ce2243c2ce96a4328fa04d8c24148c4d70d411f2eec093fbd
+tenant hash=502836444b6ee39b3ce4e61d0e6c527172a117d703f414750cb7bd7b257f7602
 ----
 [{"key":""}
-,{"key":"8b89898a89","value":"0312390a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518022200280140004a00"}
+,{"key":"8b89898a89","value":"0312450a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518022200280140004a006a0a08d7843d100118002014"}
 ,{"key":"8b898b8a89","value":"030a88030a0a64657363726970746f721803200128013a0042270a02696410011a0c08011040180030005014600020003000680070007800800100880100980100422f0a0a64657363726970746f7210021a0c08081000180030005011600020013000680070007800800100880100980100480352710a077072696d61727910011801220269642a0a64657363726970746f72300140004a10080010001a00200028003000380040005a0070027a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060026a210a0b0a0561646d696e102018200a0a0a04726f6f741020182012046e6f64651802800101880103980100b201130a077072696d61727910001a02696420012800b201240a1066616d5f325f64657363726970746f7210021a0a64657363726970746f7220022802b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300d00300"}
 ,{"key":"8b898c8a89","value":"030ac1050a0575736572731804200128013a00422d0a08757365726e616d6510011a0c0807100018003000501960002000300068007000780080010088010098010042330a0e68617368656450617373776f726410021a0c0808100018003000501160002001300068007000780080010088010098010042320a066973526f6c6510031a0c08001000180030005010600020002a0566616c73653000680070007800800100880100980100422c0a07757365725f696410041a0c080c100018003000501a60002000300068007000780080010088010098010048055290010a077072696d617279100118012208757365726e616d652a0e68617368656450617373776f72642a066973526f6c652a07757365725f6964300140004a10080010001a00200028003000380040005a007002700370047a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00102e00100e90100000000000000005a740a1175736572735f757365725f69645f696478100218012207757365725f69643004380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060036a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201240a077072696d61727910001a08757365726e616d651a07757365725f6964200120042804b2012c0a1466616d5f325f68617368656450617373776f726410021a0e68617368656450617373776f726420022802b2011c0a0c66616d5f335f6973526f6c6510031a066973526f6c6520032803b80104c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880303a80300b00300d00300"}
 ,{"key":"8b898d8a89","value":"030af7020a057a6f6e65731805200128013a0042270a02696410011a0c08011040180030005014600020003000680070007800800100880100980100422b0a06636f6e66696710021a0c080810001800300050116000200130006800700078008001008801009801004803526d0a077072696d61727910011801220269642a06636f6e666967300140004a10080010001a00200028003000380040005a0070027a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651802800101880103980100b201130a077072696d61727910001a02696420012800b2011c0a0c66616d5f325f636f6e66696710021a06636f6e66696720022802b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300d00300"}

pkg/sql/catalog/dbdesc/database_desc.go

@@ -15,6 +15,7 @@ package dbdesc
 import (
 	"fmt"
 
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb"
@@ -253,6 +254,8 @@ func (desc *immutable) ValidateSelf(vea catalog.ValidationErrorAccumulator) {
 	if desc.IsMultiRegion() {
 		desc.validateMultiRegion(vea)
 	}
+
+	desc.maybeValidateSystemDatabaseSchemaVersion(vea)
 }
 
 // validateMultiRegion performs checks specific to multi-region DBs.
@@ -267,6 +270,40 @@ func (desc *immutable) validateMultiRegion(vea catalog.ValidationErrorAccumulato
 	}
 }
 
+func (desc *immutable) maybeValidateSystemDatabaseSchemaVersion(
+	vea catalog.ValidationErrorAccumulator,
+) {
+	sv := desc.GetSystemDatabaseSchemaVersion()
+	if sv == nil {
+		return
+	}
+
+	if id := desc.GetID(); id != keys.SystemDatabaseID {
+		vea.Report(errors.AssertionFailedf(
+			`attempting to set system database schema version for non-system database descriptor (%d)`,
+			id,
+		))
+	}
+
+	binaryMinSupportedVersion := clusterversion.ByKey(clusterversion.BinaryMinSupportedVersionKey)
+	if !binaryMinSupportedVersion.LessEq(*sv) {
+		vea.Report(errors.AssertionFailedf(
+			`attempting to set system database schema version to version lower than binary min supported version (%#v): %#v`,
+			binaryMinSupportedVersion,
+			*sv,
+		))
+	}
+
+	binaryVersion := clusterversion.ByKey(clusterversion.BinaryVersionKey)
+	if !sv.LessEq(binaryVersion) {
+		vea.Report(errors.AssertionFailedf(
+			`attempting to set system database schema version to version higher than binary version (%#v): %#v`,
+			binaryVersion,
+			*sv,
+		))
+	}
+}
+
 // GetReferencedDescIDs returns the IDs of all descriptors referenced by
 // this descriptor, including itself.
 func (desc *immutable) GetReferencedDescIDs() (catalog.DescriptorIDSet, error) {

pkg/sql/catalog/descpb/BUILD.bazel

@@ -55,6 +55,7 @@ proto_library(
     deps = [
         "//pkg/config/zonepb:zonepb_proto",
         "//pkg/geo/geoindex:geoindex_proto",
+        "//pkg/roachpb:roachpb_proto",
         "//pkg/sql/catalog/catenumpb:catenumpb_proto",
         "//pkg/sql/catalog/catpb:catpb_proto",
         "//pkg/sql/schemachanger/scpb:scpb_proto",

pkg/sql/catalog/descpb/structured.proto

@@ -25,6 +25,7 @@ import "sql/schemachanger/scpb/scpb.proto";
 import "sql/types/types.proto";
 import "geo/geoindex/config.proto";
 import "gogoproto/gogo.proto";
+import "roachpb/metadata.proto";
 
 enum ConstraintValidity {
   // The constraint is valid for all rows.
@@ -1289,7 +1290,13 @@ message DatabaseDescriptor {
   // descriptor being changed as part of a declarative schema change.
   optional cockroach.sql.schemachanger.scpb.DescriptorState declarative_schema_changer_state = 12;
 
-  // Next field is 13.
+  // SystemDatabaseSchemaVersion denotes the version of the system
+  // database's schema.
+  //
+  // Note: It should only be set for the system database.
+  optional roachpb.Version system_database_schema_version = 13;
+
+  // Next field is 14.
 }
 
 // SuperRegion stores a super region configuration.

pkg/sql/catalog/systemschema/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/clusterversion",
         "//pkg/keys",
         "//pkg/security/username",
         "//pkg/sql/catalog",

pkg/sql/catalog/systemschema/system.go

@@ -16,6 +16,7 @@ import (
 	"sort"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog"
@@ -1052,6 +1053,11 @@ const (
 // SystemDatabaseName is the name of the system database.
 const SystemDatabaseName = catconstants.SystemDatabaseName
 
+// SystemDatabaseSchemaBootstrapVersion is the system database schema version
+// that should be used during bootstrap. It should be bumped up alongside any
+// upgrade that creates or modifies the schema of a system table.
+var SystemDatabaseSchemaBootstrapVersion = clusterversion.ByKey(clusterversion.V23_2_RegionaLivenessTable)
+
 // MakeSystemDatabaseDesc constructs a copy of the system database
 // descriptor.
 func MakeSystemDatabaseDesc() catalog.DatabaseDescriptor {
@@ -1063,6 +1069,7 @@ func MakeSystemDatabaseDesc() catalog.DatabaseDescriptor {
 		// Assign max privileges to root user.
 		Privileges: catpb.NewCustomSuperuserPrivilegeDescriptor(
 			priv, username.NodeUserName()),
+		SystemDatabaseSchemaVersion: &SystemDatabaseSchemaBootstrapVersion,
 	}).BuildImmutableDatabase()
 }