Merge #98440

98440: upgrades,jobs: stub the auto config runner r=stevendanna,ajwerner a=knz Epic: CRDB-23559 Informs #98431. This change is part of a larger project to perform auto-configuration in newly created clusters. One of the components will be the "auto config runner", a job which receives configuration to apply from the environment, creates other jobs to apply this configuration, and stores the execution results. This commit stubs the job, which means allocating a cluster version for it and instantiating it via a cluster migration. The job is otherwise idle at this point. Release note: None Co-authored-by: Raphael 'kena' Poss <[email protected]>
cockroachdb · Mar 16, 2023 · dbbf20d · dbbf20d
2 parents 95d6ef9 + 56c64e3
commit dbbf20d
Show file tree

Hide file tree

Showing 111 changed files with 1,236 additions and 393 deletions.
diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt
@@ -299,4 +299,4 @@ trace.opentelemetry.collector	string		address of an OpenTelemetry trace collecto
 trace.snapshot.rate	duration	0s	if non-zero, interval at which background trace snapshots are captured
 trace.span_registry.enabled	boolean	true	if set, ongoing traces can be seen at https://<ui>/#/debug/tracez
 trace.zipkin.collector	string		the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.
-version	version	1000022.2-86	set the active cluster version in the format '<major>.<minor>'
+version	version	1000022.2-90	set the active cluster version in the format '<major>.<minor>'
diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html
@@ -240,6 +240,6 @@
 <tr><td><div id="setting-trace-snapshot-rate" class="anchored"><code>trace.snapshot.rate</code></div></td><td>duration</td><td><code>0s</code></td><td>if non-zero, interval at which background trace snapshots are captured</td></tr>
 <tr><td><div id="setting-trace-span-registry-enabled" class="anchored"><code>trace.span_registry.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if set, ongoing traces can be seen at https://&lt;ui&gt;/#/debug/tracez</td></tr>
 <tr><td><div id="setting-trace-zipkin-collector" class="anchored"><code>trace.zipkin.collector</code></div></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as &lt;host&gt;:&lt;port&gt;. If no port is specified, 9411 will be used.</td></tr>
-<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000022.2-86</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td></tr>
+<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000022.2-90</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td></tr>
 </tbody>
 </table>
diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel
@@ -1420,6 +1420,7 @@ GO_TARGETS = [
     "//pkg/security/username:username_test",
     "//pkg/security:security",
     "//pkg/security:security_test",
+    "//pkg/server/autoconfig:autoconfig",
     "//pkg/server/debug/goroutineui:goroutineui",
     "//pkg/server/debug/goroutineui:goroutineui_test",
     "//pkg/server/debug/pprofui:pprofui",
@@ -2794,6 +2795,7 @@ GET_X_DATA_TARGETS = [
     "//pkg/security/sessionrevival:get_x_data",
     "//pkg/security/username:get_x_data",
     "//pkg/server:get_x_data",
+    "//pkg/server/autoconfig:get_x_data",
     "//pkg/server/debug:get_x_data",
     "//pkg/server/debug/goroutineui:get_x_data",
     "//pkg/server/debug/pprofui:get_x_data",

diff --git a/pkg/ccl/backupccl/system_schema.go b/pkg/ccl/backupccl/system_schema.go
@@ -574,6 +574,12 @@ var systemTableBackupConfiguration = map[string]systemBackupConfiguration{
 	systemschema.TenantUsageTable.GetName(): {
 		shouldIncludeInClusterBackup: optOutOfClusterBackup,
 	},
+	systemschema.SystemTenantTasksTable.GetName(): {
+		shouldIncludeInClusterBackup: optOutOfClusterBackup,
+	},
+	systemschema.SystemTaskPayloadsTable.GetName(): {
+		shouldIncludeInClusterBackup: optOutOfClusterBackup,
+	},
 	systemschema.SQLInstancesTable().GetName(): {
 		shouldIncludeInClusterBackup: optOutOfClusterBackup,
 	},

diff --git a/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant b/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant
@@ -338,12 +338,12 @@ txn_id  txn_fingerprint_id  query  implicit_txn  session_id  start_time  end_tim
 query ITTI
 SELECT range_id, start_pretty, end_pretty, lease_holder FROM crdb_internal.ranges
 ----
-59  /Tenant/10  /Tenant/11  1
+61  /Tenant/10  /Tenant/11  1
 
 query ITT
 SELECT range_id, start_pretty, end_pretty FROM crdb_internal.ranges_no_leases
 ----
-59  /Tenant/10  /Tenant/11
+61  /Tenant/10  /Tenant/11
 
 query IT
 SELECT zone_id, target FROM crdb_internal.zones ORDER BY 1

diff --git a/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error b/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error
@@ -216,7 +216,7 @@ skipif config multiregion-9node-3region-3azs-vec-off
 query I retry
 SELECT DISTINCT range_id FROM [SHOW RANGES FROM TABLE messages_rbr]
 ----
-58
+60
 
 # Update does not fail when accessing all rows in messages_rbr because lookup
 # join does not error out the lookup table in phase 1.

diff --git a/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior b/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior
@@ -262,7 +262,7 @@ ap-southeast-2  23
 query TT
 SELECT start_key, end_key FROM [SHOW RANGE FROM TABLE regional_by_row_table FOR ROW ('ap-southeast-2', 1)]
 ----
-<before:/Table/57>  …
+<before:/Table/59>  …
 
 query TIIII
 SELECT crdb_region, pk, pk2, a, b FROM regional_by_row_table
@@ -403,9 +403,10 @@ ALTER TABLE regional_by_row_table EXPERIMENTAL_RELOCATE VALUES (ARRAY[1], 'ap-so
 
 query TTTI colnames,rowsort
 SELECT start_key, end_key, replicas, lease_holder FROM [SHOW RANGES FROM INDEX regional_by_row_table@primary WITH DETAILS]
+ORDER BY 1
 ----
 start_key           end_key               replicas  lease_holder
-<before:/Table/57>  …/"\x80"/0            {1}       1
+<before:/Table/59>  …/"\x80"/0            {1}       1
 …/"\x80"/0          …/"\xc0"/0            {4}       4
 …/"\xc0"/0          <after:/Table/110/5>  {7}       7
 

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/basic b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/basic
@@ -64,6 +64,8 @@ upsert /Table/5{4-5}                       database system (host)
 upsert /Table/5{5-6}                       database system (host)
 upsert /Table/5{6-7}                       database system (host)
 upsert /Table/5{7-8}                       ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+upsert /Table/5{8-9}                       database system (host)
+upsert /Table/{59-60}                      database system (host)
 
 exec-sql
 CREATE DATABASE db;
@@ -110,6 +112,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Table/10{6-7}                             num_replicas=7 num_voters=5
 /Table/10{7-8}                             num_replicas=7
 /Table/11{2-3}                             num_replicas=7
@@ -217,6 +221,10 @@ delete /Table/5{5-6}
 upsert /Table/5{5-6}                       ttl_seconds=100 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
 delete /Table/5{6-7}
 upsert /Table/5{6-7}                       ttl_seconds=100 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+delete /Table/5{8-9}
+upsert /Table/5{8-9}                       ttl_seconds=100 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+delete /Table/{59-60}
+upsert /Table/{59-60}                      ttl_seconds=100 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
 
 state offset=5 limit=42
 ----

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/indexes b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/indexes
@@ -31,6 +31,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Table/10{6-7}                             range default
 
 exec-sql
@@ -62,6 +64,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Table/106{-/2}                            num_replicas=7
 /Table/106/{2-3}                           num_replicas=7 num_voters=5
 /Table/10{6/3-7}                           num_replicas=7
@@ -95,6 +99,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Table/106{-/2}                            ttl_seconds=3600 num_replicas=7
 /Table/106/{2-3}                           ttl_seconds=25 num_replicas=7 num_voters=5
 /Table/10{6/3-7}                           ttl_seconds=3600 num_replicas=7
@@ -118,6 +124,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Table/106{-/2}                            ttl_seconds=3600 num_replicas=9
 /Table/106/{2-3}                           ttl_seconds=25 num_replicas=9 num_voters=5
 /Table/10{6/3-7}                           ttl_seconds=3600 num_replicas=9
@@ -155,3 +163,5 @@ state offset=46
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/basic b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/basic
@@ -32,6 +32,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Tenant/10{-\x00}                          database system (tenant)
 /Tenant/11{-\x00}                          database system (tenant)
 /Tenant/12{-\x00}                          database system (tenant)
@@ -100,6 +102,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Tenant/10{-/Table/4}                      database system (tenant)
 /Tenant/10/Table/{4-5}                     database system (tenant)
 /Tenant/10/Table/{5-6}                     database system (tenant)
@@ -167,6 +171,8 @@ upsert /Tenant/10/Table/11{3-4}            range default
 state offset=81
 ----
 ...
+/Tenant/10/Table/3{3-4}                    database system (tenant)
+/Tenant/10/Table/3{4-5}                    database system (tenant)
 /Tenant/10/Table/3{5-6}                    database system (tenant)
 /Tenant/10/Table/3{6-7}                    database system (tenant)
 /Tenant/10/Table/3{7-8}                    database system (tenant)

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/protectedts b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/protectedts
@@ -29,6 +29,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Tenant/10{-\x00}                          database system (tenant)
 /Tenant/11{-\x00}                          database system (tenant)
 

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/range_tenants b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/range_tenants
@@ -48,6 +48,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Tenant/10{-\x00}                          database system (tenant)
 /Tenant/11{-\x00}                          ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
 /Tenant/12{-\x00}                          ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
@@ -74,6 +76,8 @@ state offset=47
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Tenant/10{-/Table/4}                      database system (tenant)
 /Tenant/10/Table/{4-5}                     database system (tenant)
 /Tenant/10/Table/{5-6}                     database system (tenant)
@@ -154,6 +158,8 @@ mutations discard tenant=11
 state offset=81
 ----
 ...
+/Tenant/10/Table/3{3-4}                    database system (tenant)
+/Tenant/10/Table/3{4-5}                    database system (tenant)
 /Tenant/10/Table/3{5-6}                    database system (tenant)
 /Tenant/10/Table/3{6-7}                    database system (tenant)
 /Tenant/10/Table/3{7-8}                    database system (tenant)

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/tenant_end_key_split b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/tenant_end_key_split
@@ -17,8 +17,10 @@ initialize tenant=11
 state offset=57 limit=3
 ----
 ...
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Tenant/11{-\x00}                          database system (tenant)
-/Tenant/12{-\x00}                          database system (tenant)
+...
 
 # Start the reconciliation loop for the tenant=11. It'll first clear out its own
 # first-key record and install span configs for its SQL state. It won't touch
@@ -35,8 +37,8 @@ state offset=56 limit=3
 ----
 ...
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
-/Tenant/11{-/Table/4}                      database system (tenant)
-/Tenant/11/Table/{4-5}                     database system (tenant)
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 ...
 
 # Peek near the end of the span_configurations table where tenant=11's records
@@ -45,8 +47,10 @@ state offset=56 limit=3
 state offset=99 limit=3
 ----
 ...
+/Tenant/11/Table/5{5-6}                    database system (tenant)
+/Tenant/11/Table/5{6-7}                    database system (tenant)
 /Tenant/11/Table/5{7-8}                    database system (tenant)
-/Tenant/12{-\x00}                          database system (tenant)
+...
 
 # Just another view of what the tenant's reconciler actually did. It got rid of
 # the original, single-key /Tenant/11{-\x00} record, and replaced it with
@@ -109,7 +113,7 @@ initialize tenant=10
 state offset=57 limit=3
 ----
 ...
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Tenant/10{-\x00}                          database system (tenant)
-/Tenant/11{-/Table/4}                      database system (tenant)
-/Tenant/11/Table/{4-5}                     database system (tenant)
 ...
diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/named_zones b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/named_zones
@@ -134,6 +134,8 @@ state offset=46
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Table/10{6-7}                             ttl_seconds=50
 
 # Make sure future descendants observe the same.
@@ -159,6 +161,8 @@ state offset=46
 /Table/5{5-6}                              database system (host)
 /Table/5{6-7}                              database system (host)
 /Table/5{7-8}                              ttl_seconds=3600 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+/Table/5{8-9}                              database system (host)
+/Table/{59-60}                             database system (host)
 /Table/10{6-7}                             ttl_seconds=50
 /Table/10{7-8}                             ttl_seconds=50