diff --git a/pkg/ccl/logictestccl/testdata/logic_test/multi_region b/pkg/ccl/logictestccl/testdata/logic_test/multi_region index 068c8d5ab0e2..98b8905a6102 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/multi_region +++ b/pkg/ccl/logictestccl/testdata/logic_test/multi_region @@ -544,7 +544,7 @@ ALTER DATABASE system PRIMARY REGION "ap-southeast-2" # are in the system tenant. Hence the lack of error code checking and the # regex matching both possible outcomes. skipif config multiregion-9node-3region-3azs-tenant -statement error user testuser may not modify the system database|modifying the regions of system database is not supported +statement error user testuser may not modify the system database|Modifying the regions of system database is not supported. Set up your system database as multi-region using the cluster setting `sql.multiregion.preview_multiregion_system_database.enabled` https://www.cockroachlabs.com/docs/stable/cluster-settings ALTER DATABASE system PRIMARY REGION "ap-southeast-2" user root diff --git a/pkg/ccl/multiregionccl/multiregion_system_table_test.go b/pkg/ccl/multiregionccl/multiregion_system_table_test.go index 3776bb258910..b8b4249f5a15 100644 --- a/pkg/ccl/multiregionccl/multiregion_system_table_test.go +++ b/pkg/ccl/multiregionccl/multiregion_system_table_test.go @@ -48,7 +48,7 @@ func TestMrSystemDatabase(t *testing.T) { return cs } - cluster, _, cleanup := multiregionccltestutils.TestingCreateMultiRegionCluster(t, 3, + cluster, sqlDB, cleanup := multiregionccltestutils.TestingCreateMultiRegionCluster(t, 3, base.TestingKnobs{}, multiregionccltestutils.WithSettings(makeSettings())) defer cleanup() @@ -81,6 +81,15 @@ func TestMrSystemDatabase(t *testing.T) { // okay. tDB.CheckQueryResults(t, `SELECT * FROM crdb_internal.invalid_objects`, [][]string{}) + _, err = sqlDB.Exec(`SET CLUSTER SETTING sql.multiregion.preview_multiregion_system_database.enabled = true`) + require.NoError(t, err) + _, err = sqlDB.Exec(`ALTER DATABASE system SET PRIMARY REGION "us-east1"`) + require.NoError(t, err) + _, err = sqlDB.Exec(`ALTER DATABASE system ADD REGION "us-east2"`) + require.NoError(t, err) + _, err = sqlDB.Exec(`ALTER DATABASE system ADD REGION "us-east3"`) + require.NoError(t, err) + t.Run("Sqlliveness", func(t *testing.T) { row := tDB.QueryRow(t, `SELECT crdb_region, session_id, expiration FROM system.sqlliveness LIMIT 1`) var sessionID string diff --git a/pkg/cmd/roachtest/tests/BUILD.bazel b/pkg/cmd/roachtest/tests/BUILD.bazel index 190b9778eeba..67b0076cd548 100644 --- a/pkg/cmd/roachtest/tests/BUILD.bazel +++ b/pkg/cmd/roachtest/tests/BUILD.bazel @@ -113,6 +113,7 @@ go_library( "mixed_version_job_compatibility_in_declarative_schema_changer.go", "mixed_version_multi_region.go", "mixed_version_schemachange.go", + "multi_region_system_database.go", "multiregion_leasing.go", "multitenant.go", "multitenant_distsql.go", diff --git a/pkg/cmd/roachtest/tests/multi_region_system_database.go b/pkg/cmd/roachtest/tests/multi_region_system_database.go new file mode 100644 index 000000000000..1b3d8c70878b --- /dev/null +++ b/pkg/cmd/roachtest/tests/multi_region_system_database.go @@ -0,0 +1,127 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package tests + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec" + "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" + "github.com/cockroachdb/cockroach/pkg/roachprod/install" + "github.com/stretchr/testify/require" +) + +func registerMultiRegionSystemDatabase(r registry.Registry) { + clusterSpec := r.MakeClusterSpec(3, spec.Geo(), spec.GatherCores(), spec.GCEZones("us-east1-b,us-west1-b,us-central1-b")) + r.Add(registry.TestSpec{ + Name: "multi-region-system-database", + Owner: registry.OwnerSQLFoundations, + Timeout: time.Hour * 1, + RequiresLicense: true, + Cluster: clusterSpec, + CompatibleClouds: registry.OnlyGCE, + Suites: registry.Suites(registry.Weekly), + Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { + nodes := c.Spec().NodeCount + regions := strings.Split(c.Spec().GCE.Zones, ",") + regionOnly := func(regionAndZone string) string { + r := strings.Split(regionAndZone, "-") + return r[0] + "-" + r[1] + } + t.Status("starting cluster") + c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(install.SecureOption(false))) + conn := c.Conn(ctx, t.L(), 1) + defer conn.Close() + + _, err := conn.ExecContext(ctx, "SET CLUSTER SETTING sql.multiregion.preview_multiregion_system_database.enabled = true") + require.NoError(t, err) + + _, err = conn.ExecContext(ctx, + fmt.Sprintf(`ALTER DATABASE system SET PRIMARY REGION '%s'`, regionOnly(regions[0]))) + require.NoError(t, err) + + _, err = conn.ExecContext(ctx, + fmt.Sprintf(`ALTER DATABASE system ADD REGION '%s'`, regionOnly(regions[1]))) + require.NoError(t, err) + + _, err = conn.ExecContext(ctx, + fmt.Sprintf(`ALTER DATABASE system ADD REGION '%s'`, regionOnly(regions[2]))) + require.NoError(t, err) + + //Perform rolling restart to propagate region information to non-primary nodes + for i := 2; i <= nodes; i++ { + t.WorkerStatus("stop") + err := c.StopCockroachGracefullyOnNode(ctx, t.L(), i) + if err != nil { + return + } + t.WorkerStatus("start") + startOpts := option.DefaultStartOpts() + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(install.SecureOption(false)), c.Node(i)) + } + + //Check system.lease table to ensure that region information for each node is correct + rows, err := conn.Query("SELECT DISTINCT sql_instance_id, crdb_region FROM system.lease") + require.NoError(t, err) + + nodeToRegionName := make(map[int]string) + for rows.Next() { + var sqlInstanceID int + var crdbRegion string + require.NoError(t, rows.Scan(&sqlInstanceID, &crdbRegion)) + nodeToRegionName[sqlInstanceID] = crdbRegion + } + + for node, regionName := range nodeToRegionName { + require.Equal(t, regionOnly(regions[node-1]), regionName) + } + + //Intentionally tear down nodes and ensure that everything is still working + chaosTest := func() { + //Random operations on user-created table + _, err := conn.Exec(`CREATE TABLE foo (key INT PRIMARY KEY)`) + if err != nil { + return + } + defer func() { + _, err := conn.Exec(`DROP TABLE foo`) + if err != nil { + return + } + }() + _, err = conn.Exec(`INSERT INTO foo VALUES (1), (2), (3)`) + require.NoError(t, err) + row := conn.QueryRow(`SELECT * FROM foo LIMIT 1`) + var rowPK int + require.NoError(t, row.Scan(&rowPK)) + require.Equal(t, 1, rowPK) + } + + for i := 2; i <= nodes; i++ { + t.WorkerStatus("stop") + c.Run(ctx, option.WithNodes(c.Node(i)), "killall -9 cockroach") + + t.WorkerStatus("chaos testing") + chaosTest() + + t.WorkerStatus("start") + startOpts := option.DefaultStartOpts() + c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(install.SecureOption(false)), c.Node(i)) + } + }, + }) +} diff --git a/pkg/cmd/roachtest/tests/registry.go b/pkg/cmd/roachtest/tests/registry.go index be40147c0f9e..f785131507b3 100644 --- a/pkg/cmd/roachtest/tests/registry.go +++ b/pkg/cmd/roachtest/tests/registry.go @@ -154,4 +154,5 @@ func RegisterTests(r registry.Registry) { registerYCSB(r) registerDeclarativeSchemaChangerJobCompatibilityInMixedVersion(r) registerMultiRegionMixedVersion(r) + registerMultiRegionSystemDatabase(r) } diff --git a/pkg/sql/alter_database.go b/pkg/sql/alter_database.go index 07348685b54e..ae0bef4bf727 100644 --- a/pkg/sql/alter_database.go +++ b/pkg/sql/alter_database.go @@ -269,6 +269,11 @@ func (n *alterDatabaseAddRegionNode) startExec(params runParams) error { if err := params.p.setSystemDatabaseSurvival(params.ctx); err != nil { return err } + + params.p.BufferClientNotice( + params.ctx, + pgnotice.Newf("Rolling restart is recommended after adding a region to system database in order to propogate region information."), + ) } // Validate the type descriptor after the changes. We have to do this explicitly here, because @@ -586,10 +591,14 @@ func (p *planner) checkPrivilegesForMultiRegionOp( // multi-region primitives in the system database. For now, we also allow // root to perform the various operations to enable testing. if desc.GetID() == keys.SystemDatabaseID { - if p.execCfg.Codec.ForSystemTenant() { + if multiRegionSystemDatabase := sqlclustersettings.MultiRegionSystemDatabaseEnabled.Get(&p.execCfg.Settings.SV); !multiRegionSystemDatabase && + p.execCfg.Codec.ForSystemTenant() { return pgerror.Newf( pgcode.FeatureNotSupported, - "modifying the regions of system database is not supported", + "Modifying the regions of system database is not supported. "+ + "Set up your system database as multi-region using the cluster setting "+ + "`sql.multiregion.preview_multiregion_system_database.enabled` "+ + "https://www.cockroachlabs.com/docs/stable/cluster-settings.", ) } if u := p.SessionData().User(); !u.IsNodeUser() && !u.IsRootUser() { diff --git a/pkg/sql/sqlclustersettings/clustersettings.go b/pkg/sql/sqlclustersettings/clustersettings.go index b96a3df09736..e51c2413c48b 100644 --- a/pkg/sql/sqlclustersettings/clustersettings.go +++ b/pkg/sql/sqlclustersettings/clustersettings.go @@ -79,6 +79,15 @@ var SecondaryTenantsAllZoneConfigsEnabled = settings.RegisterBoolSetting( true, ) +// MultiRegionSystemDatabaseEnabled controls if system tenants are allowed +// to be set up to be multi-region. +var MultiRegionSystemDatabaseEnabled = settings.RegisterBoolSetting( + settings.SystemVisible, + "sql.multiregion.preview_multiregion_system_database.enabled", + "enable option to set up system database as multi-region", + false, +) + // RequireSystemTenantOrClusterSetting returns a setting disabled error if // executed from inside a secondary tenant that does not have the specified // cluster setting.