-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
slstorage: support regional by row system.sqlliveness
The COCKROACH_MR_SYSTEM_DATABASE environment variable was introduced to control features added as part of #85736. The variable will be removed once migrations and version gates are implemented for the multi-region system database optimizations. The sqlliveness table now supports an index format compatible with regional by row tables. Multi region serverless is the motivation for this change. When a sql server starts up, it must write its session to the sqlliveness table. The remote session write can add ~400ms to a servers startup time. ``` $ COCKROACH_MR_SYSTEM_DATABASE=1 ./cockroach-short demo [email protected]:26257/movr> SELECT * FROM system.sqlliveness; crdb_region | session_uuid | expiration --------------+------------------------------------+--------------------------------- \x80 | \x5895d5d121984d86bcb81d6a89d5635d | 1667511355275484444.0000000000 $ COCKROACH_MR_SYSTEM_DATABASE=0 ./cockroach-short demo [email protected]:26257/movr> select * from system.sqlliveness; session_id | expiration -------------------------------------------+--------------------------------- \x0101805fc92bdb0e8d4cb4aec3e1527a3052b8 | 1667511392212781946.0000000000 ``` Part of #85736 Release note: None
- Loading branch information
1 parent
2c6b3c0
commit 68b34ba
Showing
10 changed files
with
516 additions
and
104 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
116 changes: 116 additions & 0 deletions
116
pkg/ccl/multiregionccl/multiregion_system_table_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| // Copyright 2022 The Cockroach Authors. | ||
| // | ||
| // Licensed as a CockroachDB Enterprise file under the Cockroach Community | ||
| // License (the "License"); you may not use this file except in compliance with | ||
| // the License. You may obtain a copy of the License at | ||
| // | ||
| // https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt | ||
|
|
||
| package multiregionccl | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/cockroachdb/apd/v3" | ||
| "github.com/cockroachdb/cockroach/pkg/base" | ||
| "github.com/cockroachdb/cockroach/pkg/ccl/multiregionccl/multiregionccltestutils" | ||
| "github.com/cockroachdb/cockroach/pkg/keys" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/enum" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage" | ||
| "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" | ||
| "github.com/cockroachdb/cockroach/pkg/util/hlc" | ||
| "github.com/cockroachdb/cockroach/pkg/util/leaktest" | ||
| "github.com/cockroachdb/cockroach/pkg/util/log" | ||
| "github.com/cockroachdb/cockroach/pkg/util/timeutil" | ||
| "github.com/cockroachdb/cockroach/pkg/util/uuid" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func createSqllivenessTable( | ||
| t *testing.T, db *sqlutils.SQLRunner, dbName string, | ||
| ) (tableID descpb.ID) { | ||
| t.Helper() | ||
| db.Exec(t, fmt.Sprintf(` | ||
| CREATE DATABASE IF NOT EXISTS "%s" | ||
| WITH PRIMARY REGION "us-east1" | ||
| REGIONS "us-east1", "us-east2", "us-east3" | ||
| `, dbName)) | ||
|
|
||
| // expiration needs to be column 2. slstorage.Table assumes the column id. | ||
| // session_uuid and crdb_region are identified by their location in the | ||
| // primary key. | ||
| db.Exec(t, fmt.Sprintf(` | ||
| CREATE TABLE "%s".sqlliveness ( | ||
| session_uuid BYTES NOT NULL, | ||
| expiration DECIMAL NOT NULL, | ||
| crdb_region "%s".public.crdb_internal_region, | ||
| PRIMARY KEY(crdb_region, session_uuid) | ||
| ) LOCALITY REGIONAL BY ROW; | ||
| `, dbName, dbName)) | ||
| db.QueryRow(t, ` | ||
| select u.id | ||
| from system.namespace t | ||
| join system.namespace u | ||
| on t.id = u."parentID" | ||
| where t.name = $1 and u.name = $2`, | ||
| dbName, "sqlliveness").Scan(&tableID) | ||
| return tableID | ||
| } | ||
|
|
||
| func TestRbrSqllivenessTable(t *testing.T) { | ||
| defer leaktest.AfterTest(t)() | ||
| defer log.Scope(t).Close(t) | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| cluster, sqlDB, cleanup := multiregionccltestutils.TestingCreateMultiRegionCluster(t, 3, base.TestingKnobs{}) | ||
| defer cleanup() | ||
| kvDB := cluster.Servers[0].DB() | ||
| settings := cluster.Servers[0].Cfg.Settings | ||
| stopper := cluster.Servers[0].Stopper() | ||
|
|
||
| tDB := sqlutils.MakeSQLRunner(sqlDB) | ||
|
|
||
| t0 := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) | ||
| timeSource := timeutil.NewManualTime(t0) | ||
| clock := hlc.NewClock(timeSource, base.DefaultMaxClockOffset) | ||
|
|
||
| setup := func(t *testing.T) *slstorage.Storage { | ||
| dbName := t.Name() | ||
| tableID := createSqllivenessTable(t, tDB, dbName) | ||
| clock := hlc.NewClock(timeSource, base.DefaultMaxClockOffset) | ||
| var ambientCtx log.AmbientContext | ||
| return slstorage.NewTestingStorage(ambientCtx, stopper, clock, kvDB, keys.SystemSQLCodec, settings, | ||
| tableID, timeSource.NewTimer) | ||
| } | ||
|
|
||
| t.Run("SqlRead", func(t *testing.T) { | ||
| storage := setup(t) | ||
|
|
||
| initialUUID := uuid.MakeV4() | ||
| session, err := slstorage.MakeSessionID(enum.One, initialUUID) | ||
| require.NoError(t, err) | ||
|
|
||
| writeExpiration := clock.Now().Add(10, 00) | ||
| require.NoError(t, storage.Insert(ctx, session, writeExpiration)) | ||
|
|
||
| var sessionUUID string | ||
| var crdbRegion string | ||
| var rawExpiration apd.Decimal | ||
|
|
||
| row := tDB.QueryRow(t, fmt.Sprintf(`SELECT crdb_region, session_uuid, expiration FROM "%s".sqlliveness`, t.Name())) | ||
| row.Scan(&crdbRegion, &sessionUUID, &rawExpiration) | ||
|
|
||
| require.Contains(t, []string{"us-east1", "us-east2", "us-east3"}, crdbRegion) | ||
| require.Equal(t, sessionUUID, string(initialUUID.GetBytes())) | ||
|
|
||
| readExpiration, err := hlc.DecimalToHLC(&rawExpiration) | ||
| require.NoError(t, err) | ||
|
|
||
| require.Equal(t, writeExpiration, readExpiration) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.