Merge cockroachdb#62053

62053: sql: Add system table join_tokens, and create_join_tokens() builtin function r=knz a=itsbilal This change adds a new system table, `join_tokens`, for the exclusive use of storing join tokens. This is necessary as we need guaranteed at-most-once semantics with these, which transactions give us pretty easily. A related migration is also added to create said table This change also adds a new builtin function, `crdb_internal.create_join_token()` that creates and persists a join token in that table. Currently, there's no mechanism to remove expired join tokens. See RFC cockroachdb#51991. Part of cockroachdb#60632. Release note (general change): Add `crdb_internal.create_join_token()` sql builtin function to create join tokens for use when joining new nodes to a secure cluster. This functionality is hidden behind a feature flag. Co-authored-by: Bilal Akhtar <[email protected]>
nvanbenschoten · Mar 29, 2021 · 0e70529 · 0e70529
2 parents 915fe3f + 3a44b45
commit 0e70529
Show file tree

Hide file tree

Showing 46 changed files with 505 additions and 194 deletions.
diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt
@@ -105,4 +105,4 @@ trace.datadog.project	string	CockroachDB	the project under which traces will be
 trace.debug.enable	boolean	false	if set, traces for recent requests can be seen at https://<ui>/debug/requests
 trace.lightstep.token	string		if set, traces go to Lightstep using this token
 trace.zipkin.collector	string		if set, traces go to the given Zipkin instance (example: '127.0.0.1:9411'). Only one tracer can be configured at a time.
-version	version	20.2-48	set the active cluster version in the format '<major>.<minor>'
+version	version	20.2-50	set the active cluster version in the format '<major>.<minor>'
diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html
@@ -107,6 +107,6 @@
 <tr><td><code>trace.debug.enable</code></td><td>boolean</td><td><code>false</code></td><td>if set, traces for recent requests can be seen at https://<ui>/debug/requests</td></tr>
 <tr><td><code>trace.lightstep.token</code></td><td>string</td><td><code></code></td><td>if set, traces go to Lightstep using this token</td></tr>
 <tr><td><code>trace.zipkin.collector</code></td><td>string</td><td><code></code></td><td>if set, traces go to the given Zipkin instance (example: '127.0.0.1:9411'). Only one tracer can be configured at a time.</td></tr>
-<tr><td><code>version</code></td><td>version</td><td><code>20.2-48</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
+<tr><td><code>version</code></td><td>version</td><td><code>20.2-50</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
 </tbody>
 </table>
diff --git a/docs/generated/sql/functions.md b/docs/generated/sql/functions.md
@@ -2659,6 +2659,8 @@ SELECT * FROM crdb_internal.check_consistency(true, ‘\x02’, ‘\x04’)</p>
 </span></td></tr>
 <tr><td><a name="crdb_internal.completed_migrations"></a><code>crdb_internal.completed_migrations() &rarr; <a href="string.html">string</a>[]</code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
 </span></td></tr>
+<tr><td><a name="crdb_internal.create_join_token"></a><code>crdb_internal.create_join_token() &rarr; <a href="string.html">string</a></code></td><td><span class="funcdesc"><p>Creates a join token for use when adding a new node to a secure cluster.</p>
+</span></td></tr>
 <tr><td><a name="crdb_internal.encode_key"></a><code>crdb_internal.encode_key(table_id: <a href="int.html">int</a>, index_id: <a href="int.html">int</a>, row_tuple: anyelement) &rarr; <a href="bytes.html">bytes</a></code></td><td><span class="funcdesc"><p>Generate the key for a row on a particular table and index.</p>
 </span></td></tr>
 <tr><td><a name="crdb_internal.force_assertion_error"></a><code>crdb_internal.force_assertion_error(msg: <a href="string.html">string</a>) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>

diff --git a/pkg/ccl/backupccl/system_schema.go b/pkg/ccl/backupccl/system_schema.go
@@ -244,6 +244,9 @@ var systemTableBackupConfiguration = map[string]systemBackupConfiguration{
 	systemschema.MigrationsTable.GetName(): {
 		shouldIncludeInClusterBackup: optOutOfClusterBackup,
 	},
+	systemschema.JoinTokensTable.GetName(): {
+		shouldIncludeInClusterBackup: optOutOfClusterBackup,
+	},
 }
 
 // GetSystemTablesToIncludeInClusterBackup returns a set of system table names that

diff --git a/pkg/cli/testdata/doctor/testcluster b/pkg/cli/testdata/doctor/testcluster
@@ -1,7 +1,7 @@
 doctor cluster
 ----
 debug doctor cluster
-Examining 35 descriptors and 36 namespace entries...
+Examining 36 descriptors and 37 namespace entries...
   ParentID  50, ParentSchemaID 29: relation "foo" (53): expected matching namespace entry, found none
 Examining 1 running jobs...
 ERROR: validation failed
diff --git a/pkg/cli/testdata/zip/partial1 b/pkg/cli/testdata/zip/partial1
@@ -61,7 +61,7 @@ requesting goroutine files for node 1... writing: debug/nodes/1/goroutines.err.t
   ^- resulted in ...
 requesting log file ...
 requesting log file ...
-requesting ranges... 36 found
+requesting ranges... 37 found
 writing: debug/nodes/1/ranges/1.json
 writing: debug/nodes/1/ranges/2.json
 writing: debug/nodes/1/ranges/3.json
@@ -98,6 +98,7 @@ writing: debug/nodes/1/ranges/33.json
 writing: debug/nodes/1/ranges/34.json
 writing: debug/nodes/1/ranges/35.json
 writing: debug/nodes/1/ranges/36.json
+writing: debug/nodes/1/ranges/37.json
 writing: debug/nodes/2/status.json
 using SQL connection URL for node 2: postgresql://...
 retrieving SQL data for crdb_internal.feature_usage... writing: debug/nodes/2/crdb_internal.feature_usage.txt
@@ -202,7 +203,7 @@ requesting goroutine files for node 3... writing: debug/nodes/3/goroutines.err.t
   ^- resulted in ...
 requesting log file ...
 requesting log file ...
-requesting ranges... 36 found
+requesting ranges... 37 found
 writing: debug/nodes/3/ranges/1.json
 writing: debug/nodes/3/ranges/2.json
 writing: debug/nodes/3/ranges/3.json
@@ -239,14 +240,15 @@ writing: debug/nodes/3/ranges/33.json
 writing: debug/nodes/3/ranges/34.json
 writing: debug/nodes/3/ranges/35.json
 writing: debug/nodes/3/ranges/36.json
+writing: debug/nodes/3/ranges/37.json
 doctor examining cluster...writing: debug/reports/doctor.txt
 requesting list of SQL databases... 3 found
 requesting database details for defaultdb... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for postgres... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for system... writing: debug/schema/[email protected]
-30 tables found
+31 tables found
 requesting table details for system.public.namespace... writing: debug/schema/system/public_namespace.json
 requesting table details for system.public.descriptor... writing: debug/schema/system/public_descriptor.json
 requesting table details for system.public.users... writing: debug/schema/system/public_users.json
@@ -277,5 +279,6 @@ requesting table details for system.public.statement_diagnostics... writing: deb
 requesting table details for system.public.scheduled_jobs... writing: debug/schema/system/public_scheduled_jobs.json
 requesting table details for system.public.sqlliveness... writing: debug/schema/system/public_sqlliveness.json
 requesting table details for system.public.migrations... writing: debug/schema/system/public_migrations.json
+requesting table details for system.public.join_tokens... writing: debug/schema/system/public_join_tokens.json
 writing: debug/pprof-summary.sh
 writing: debug/hot-ranges.sh
diff --git a/pkg/cli/testdata/zip/partial1_excluded b/pkg/cli/testdata/zip/partial1_excluded
@@ -61,7 +61,7 @@ requesting goroutine files for node 1... writing: debug/nodes/1/goroutines.err.t
   ^- resulted in ...
 requesting log file ...
 requesting log file ...
-requesting ranges... 36 found
+requesting ranges... 37 found
 writing: debug/nodes/1/ranges/1.json
 writing: debug/nodes/1/ranges/2.json
 writing: debug/nodes/1/ranges/3.json
@@ -98,6 +98,7 @@ writing: debug/nodes/1/ranges/33.json
 writing: debug/nodes/1/ranges/34.json
 writing: debug/nodes/1/ranges/35.json
 writing: debug/nodes/1/ranges/36.json
+writing: debug/nodes/1/ranges/37.json
 writing: debug/nodes/2.skipped
 writing: debug/nodes/3/status.json
 using SQL connection URL for node 3: postgresql://...
@@ -130,7 +131,7 @@ requesting goroutine files for node 3... writing: debug/nodes/3/goroutines.err.t
   ^- resulted in ...
 requesting log file ...
 requesting log file ...
-requesting ranges... 36 found
+requesting ranges... 37 found
 writing: debug/nodes/3/ranges/1.json
 writing: debug/nodes/3/ranges/2.json
 writing: debug/nodes/3/ranges/3.json
@@ -167,14 +168,15 @@ writing: debug/nodes/3/ranges/33.json
 writing: debug/nodes/3/ranges/34.json
 writing: debug/nodes/3/ranges/35.json
 writing: debug/nodes/3/ranges/36.json
+writing: debug/nodes/3/ranges/37.json
 doctor examining cluster...writing: debug/reports/doctor.txt
 requesting list of SQL databases... 3 found
 requesting database details for defaultdb... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for postgres... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for system... writing: debug/schema/[email protected]
-30 tables found
+31 tables found
 requesting table details for system.public.namespace... writing: debug/schema/system/public_namespace.json
 requesting table details for system.public.descriptor... writing: debug/schema/system/public_descriptor.json
 requesting table details for system.public.users... writing: debug/schema/system/public_users.json
@@ -205,5 +207,6 @@ requesting table details for system.public.statement_diagnostics... writing: deb
 requesting table details for system.public.scheduled_jobs... writing: debug/schema/system/public_scheduled_jobs.json
 requesting table details for system.public.sqlliveness... writing: debug/schema/system/public_sqlliveness.json
 requesting table details for system.public.migrations... writing: debug/schema/system/public_migrations.json
+requesting table details for system.public.join_tokens... writing: debug/schema/system/public_join_tokens.json
 writing: debug/pprof-summary.sh
 writing: debug/hot-ranges.sh
diff --git a/pkg/cli/testdata/zip/partial2 b/pkg/cli/testdata/zip/partial2
@@ -61,7 +61,7 @@ requesting goroutine files for node 1... writing: debug/nodes/1/goroutines.err.t
   ^- resulted in ...
 requesting log file ...
 requesting log file ...
-requesting ranges... 36 found
+requesting ranges... 37 found
 writing: debug/nodes/1/ranges/1.json
 writing: debug/nodes/1/ranges/2.json
 writing: debug/nodes/1/ranges/3.json
@@ -98,6 +98,7 @@ writing: debug/nodes/1/ranges/33.json
 writing: debug/nodes/1/ranges/34.json
 writing: debug/nodes/1/ranges/35.json
 writing: debug/nodes/1/ranges/36.json
+writing: debug/nodes/1/ranges/37.json
 writing: debug/nodes/3/status.json
 using SQL connection URL for node 3: postgresql://...
 retrieving SQL data for crdb_internal.feature_usage... writing: debug/nodes/3/crdb_internal.feature_usage.txt
@@ -129,7 +130,7 @@ requesting goroutine files for node 3... writing: debug/nodes/3/goroutines.err.t
   ^- resulted in ...
 requesting log file ...
 requesting log file ...
-requesting ranges... 36 found
+requesting ranges... 37 found
 writing: debug/nodes/3/ranges/1.json
 writing: debug/nodes/3/ranges/2.json
 writing: debug/nodes/3/ranges/3.json
@@ -166,14 +167,15 @@ writing: debug/nodes/3/ranges/33.json
 writing: debug/nodes/3/ranges/34.json
 writing: debug/nodes/3/ranges/35.json
 writing: debug/nodes/3/ranges/36.json
+writing: debug/nodes/3/ranges/37.json
 doctor examining cluster...writing: debug/reports/doctor.txt
 requesting list of SQL databases... 3 found
 requesting database details for defaultdb... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for postgres... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for system... writing: debug/schema/[email protected]
-30 tables found
+31 tables found
 requesting table details for system.public.namespace... writing: debug/schema/system/public_namespace.json
 requesting table details for system.public.descriptor... writing: debug/schema/system/public_descriptor.json
 requesting table details for system.public.users... writing: debug/schema/system/public_users.json
@@ -204,5 +206,6 @@ requesting table details for system.public.statement_diagnostics... writing: deb
 requesting table details for system.public.scheduled_jobs... writing: debug/schema/system/public_scheduled_jobs.json
 requesting table details for system.public.sqlliveness... writing: debug/schema/system/public_sqlliveness.json
 requesting table details for system.public.migrations... writing: debug/schema/system/public_migrations.json
+requesting table details for system.public.join_tokens... writing: debug/schema/system/public_join_tokens.json
 writing: debug/pprof-summary.sh
 writing: debug/hot-ranges.sh
diff --git a/pkg/cli/testdata/zip/specialnames b/pkg/cli/testdata/zip/specialnames
@@ -22,7 +22,7 @@ requesting table details for defaultdb.public."../system"... writing: debug/sche
 requesting database details for postgres... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for system... writing: debug/schema/[email protected]
-30 tables found
+31 tables found
 requesting table details for system.public.namespace... writing: debug/schema/system-1/public_namespace.json
 requesting table details for system.public.descriptor... writing: debug/schema/system-1/public_descriptor.json
 requesting table details for system.public.users... writing: debug/schema/system-1/public_users.json
@@ -53,3 +53,4 @@ requesting table details for system.public.statement_diagnostics... writing: deb
 requesting table details for system.public.scheduled_jobs... writing: debug/schema/system-1/public_scheduled_jobs.json
 requesting table details for system.public.sqlliveness... writing: debug/schema/system-1/public_sqlliveness.json
 requesting table details for system.public.migrations... writing: debug/schema/system-1/public_migrations.json
+requesting table details for system.public.join_tokens... writing: debug/schema/system-1/public_join_tokens.json
diff --git a/pkg/cli/testdata/zip/testzip b/pkg/cli/testdata/zip/testzip
@@ -60,7 +60,7 @@ requesting heap profile for node 1... writing: debug/nodes/1/heap.pprof
 requesting heap files for node 1... ? found
 requesting goroutine files for node 1... 0 found
 requesting log file ...
-requesting ranges... 36 found
+requesting ranges... 37 found
 writing: debug/nodes/1/ranges/1.json
 writing: debug/nodes/1/ranges/2.json
 writing: debug/nodes/1/ranges/3.json
@@ -97,14 +97,15 @@ writing: debug/nodes/1/ranges/33.json
 writing: debug/nodes/1/ranges/34.json
 writing: debug/nodes/1/ranges/35.json
 writing: debug/nodes/1/ranges/36.json
+writing: debug/nodes/1/ranges/37.json
 doctor examining cluster...writing: debug/reports/doctor.txt
 requesting list of SQL databases... 3 found
 requesting database details for defaultdb... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for postgres... writing: debug/schema/[email protected]
 0 tables found
 requesting database details for system... writing: debug/schema/[email protected]
-30 tables found
+31 tables found
 requesting table details for system.public.namespace... writing: debug/schema/system/public_namespace.json
 requesting table details for system.public.descriptor... writing: debug/schema/system/public_descriptor.json
 requesting table details for system.public.users... writing: debug/schema/system/public_users.json
@@ -135,5 +136,6 @@ requesting table details for system.public.statement_diagnostics... writing: deb
 requesting table details for system.public.scheduled_jobs... writing: debug/schema/system/public_scheduled_jobs.json
 requesting table details for system.public.sqlliveness... writing: debug/schema/system/public_sqlliveness.json
 requesting table details for system.public.migrations... writing: debug/schema/system/public_migrations.json
+requesting table details for system.public.join_tokens... writing: debug/schema/system/public_join_tokens.json
 writing: debug/pprof-summary.sh
 writing: debug/hot-ranges.sh
diff --git a/pkg/clusterversion/cockroach_versions.go b/pkg/clusterversion/cockroach_versions.go
@@ -275,6 +275,9 @@ const (
 	// ProtectedTsMetaPrivilegesMigration is for the migration which fixes the
 	// privileges of the protected_ts_meta system table.
 	ProtectedTsMetaPrivilegesMigration
+	// JoinTokensTable adds the system table for storing ephemeral generated
+	// join tokens.
+	JoinTokensTable
 
 	// Step (1): Add new versions here.
 )
@@ -477,6 +480,10 @@ var versionsSingleton = keyedVersions([]keyedVersion{
 		Key:     ProtectedTsMetaPrivilegesMigration,
 		Version: roachpb.Version{Major: 20, Minor: 2, Internal: 48},
 	},
+	{
+		Key:     JoinTokensTable,
+		Version: roachpb.Version{Major: 20, Minor: 2, Internal: 50},
+	},
 	// Step (2): Add new versions here.
 })
 

diff --git a/pkg/clusterversion/key_string.go b/pkg/clusterversion/key_string.go
diff --git a/pkg/keys/constants.go b/pkg/keys/constants.go
@@ -403,6 +403,7 @@ const (
 	TenantsRangesID                     = 38 // pseudo
 	SqllivenessID                       = 39
 	MigrationsID                        = 40
+	JoinTokensTableID                   = 41
 
 	// CommentType is type for system.comments
 	DatabaseCommentType = 0

diff --git a/pkg/migration/migrations/BUILD.bazel b/pkg/migration/migrations/BUILD.bazel
@@ -4,6 +4,7 @@ go_library(
     name = "migrations",
     srcs = [
         "foreign_key_representation_upgrade.go",
+        "join_tokens.go",
         "migrations.go",
         "migrations_table.go",
         "namespace_migration.go",

diff --git a/pkg/migration/migrations/join_tokens.go b/pkg/migration/migrations/join_tokens.go
@@ -0,0 +1,28 @@
+// Copyright 2021 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package migrations
+
+import (
+	"context"
+
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
+	"github.com/cockroachdb/cockroach/pkg/migration"
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema"
+	"github.com/cockroachdb/cockroach/pkg/sqlmigrations"
+)
+
+func joinTokensTableMigration(
+	ctx context.Context, _ clusterversion.ClusterVersion, d migration.SQLDeps,
+) error {
+	return sqlmigrations.CreateSystemTable(
+		ctx, d.DB, d.Codec, d.Settings, systemschema.JoinTokensTable,
+	)
+}
diff --git a/pkg/migration/migrations/migrations.go b/pkg/migration/migrations/migrations.go
@@ -62,6 +62,11 @@ var migrations = []migration.Migration{
 		toCV(clusterversion.ProtectedTsMetaPrivilegesMigration),
 		protectedTsMetaPrivilegesMigration,
 	),
+	migration.NewSQLMigration(
+		"add the systems.join_tokens table",
+		toCV(clusterversion.JoinTokensTable),
+		joinTokensTableMigration,
+	),
 }
 
 func init() {