sql: add locality to system.sql_instances table

This PR adds the column `locality` to the `system.sql_instances` table
that contains the locality (e.g., region) of a SQL instance. The encoded
locality is a JSONB representing the `roachpb.Locality` that may have
been provided when the instance was created.

This change also pipes the locality through `InstanceInfo`. This will
allow us to determine and use locality information of other SQL
instances, e.g. in DistSQL for multi-tenant locality-awareness
distribution planning.

Informs: cockroachdb#80678

Release note (sql change): Table `system.sql_instances` has a new
column, `locality`, that stores the locality of a SQL instance if it was
provided when the instance was started. This exposes a SQL instance's
locality to other instances in the cluster for query planning.

docs/generated/settings/settings-for-tenants.txt

@@ -281,4 +281,4 @@ trace.jaeger.agent	string		the address of a Jaeger agent to receive traces using
 trace.opentelemetry.collector	string		address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.
 trace.span_registry.enabled	boolean	true	if set, ongoing traces can be seen at https://<ui>/#/debug/tracez
 trace.zipkin.collector	string		the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.
-version	version	22.1-20	set the active cluster version in the format '<major>.<minor>'
+version	version	22.1-22	set the active cluster version in the format '<major>.<minor>'

docs/generated/settings/settings.html

@@ -212,6 +212,6 @@
 <tr><td><code>trace.opentelemetry.collector</code></td><td>string</td><td><code></code></td><td>address of an OpenTelemetry trace collector to receive traces using the otel gRPC protocol, as <host>:<port>. If no port is specified, 4317 will be used.</td></tr>
 <tr><td><code>trace.span_registry.enabled</code></td><td>boolean</td><td><code>true</code></td><td>if set, ongoing traces can be seen at https://<ui>/#/debug/tracez</td></tr>
 <tr><td><code>trace.zipkin.collector</code></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.</td></tr>
-<tr><td><code>version</code></td><td>version</td><td><code>22.1-20</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
+<tr><td><code>version</code></td><td>version</td><td><code>22.1-22</code></td><td>set the active cluster version in the format '<major>.<minor>'</td></tr>
 </tbody>
 </table>

pkg/clusterversion/cockroach_versions.go

@@ -381,6 +381,10 @@ const (
 	// Previously, SSTs containing these could error.
 	AddSSTableTombstones
 
+	// AlterSystemSQLInstancesAddLocality adds a locality column to the
+	// system.sql_instances table.
+	AlterSystemSQLInstancesAddLocality
+
 	// *************************************************
 	// Step (1): Add new versions here.
 	// Do not add new versions to a patch release.
@@ -673,6 +677,10 @@ var versionsSingleton = keyedVersions{
 		Key:     AddSSTableTombstones,
 		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 20},
 	},
+	{
+		Key:     AlterSystemSQLInstancesAddLocality,
+		Version: roachpb.Version{Major: 22, Minor: 1, Internal: 22},
+	},
 
 	// *************************************************
 	// Step (2): Add new versions here.

pkg/server/server_sql.go

@@ -406,7 +406,7 @@ func newSQLServer(ctx context.Context, cfg sqlServerArgs) (*SQLServer, error) {
 		cfg.stopper, cfg.clock, cfg.db, codec, cfg.Settings, sqllivenessKnobs,
 	)
 	cfg.sqlInstanceProvider = instanceprovider.New(
-		cfg.stopper, cfg.db, codec, cfg.sqlLivenessProvider, cfg.advertiseAddr, cfg.rangeFeedFactory, cfg.clock,
+		cfg.stopper, cfg.db, codec, cfg.sqlLivenessProvider, cfg.advertiseAddr, cfg.Locality, cfg.rangeFeedFactory, cfg.clock,
 	)
 
 	if !codec.ForSystemTenant() {

pkg/sql/catalog/systemschema/system.go

@@ -639,8 +639,9 @@ CREATE TABLE system.sql_instances (
     id           INT NOT NULL,
     addr         STRING,
     session_id   BYTES,
+    locality     JSONB,
     CONSTRAINT "primary" PRIMARY KEY (id),
-    FAMILY "primary" (id, addr, session_id)
+    FAMILY "primary" (id, addr, session_id, locality)
 )`
 
 	SpanConfigurationsTableSchema = `
@@ -2281,7 +2282,7 @@ var (
 
 	// SQLInstancesTable is the descriptor for the sqlinstances table
 	// It stores information about all the SQL instances for a tenant
-	// and their associated session and address information.
+	// and their associated session, locality, and address information.
 	SQLInstancesTable = registerSystemTable(
 		SQLInstancesTableSchema,
 		systemTable(
@@ -2291,13 +2292,14 @@ var (
 				{Name: "id", ID: 1, Type: types.Int, Nullable: false},
 				{Name: "addr", ID: 2, Type: types.String, Nullable: true},
 				{Name: "session_id", ID: 3, Type: types.Bytes, Nullable: true},
+				{Name: "locality", ID: 4, Type: types.Jsonb, Nullable: true},
 			},
 			[]descpb.ColumnFamilyDescriptor{
 				{
 					Name:            "primary",
 					ID:              0,
-					ColumnNames:     []string{"id", "addr", "session_id"},
-					ColumnIDs:       []descpb.ColumnID{1, 2, 3},
+					ColumnNames:     []string{"id", "addr", "session_id", "locality"},
+					ColumnIDs:       []descpb.ColumnID{1, 2, 3, 4},
 					DefaultColumnID: 0,
 				},
 			},

pkg/sql/catalog/systemschema_test/testdata/bootstrap

@@ -346,6 +346,7 @@ CREATE TABLE public.sql_instances (
 	id INT8 NOT NULL,
 	addr STRING NULL,
 	session_id BYTES NULL,
+	locality STRING NULL,
 	CONSTRAINT "primary" PRIMARY KEY (id ASC)
 );
 CREATE TABLE public.span_configurations (

pkg/sql/logictest/testdata/logic_test/distsql_tenant_locality

@@ -31,4 +31,12 @@ SELECT start_pretty, lease_holder FROM crdb_internal.ranges WHERE start_pretty L
 
 # TODO(harding): Once locality-aware distribution is implemented, run queries in
 # the secondary tenant.
-#user root
+user root
+
+# Check sql instance locality in the secondary tenant.
+query IT
+SELECT id, locality FROM system.sql_instances
+----
+1  {"Tiers": "region=test"}
+2  {"Tiers": "region=test1"}
+3  {"Tiers": "region=test2"}

pkg/sql/logictest/testdata/logic_test/information_schema

@@ -2135,6 +2135,7 @@ system         pg_extension  spatial_ref_sys                  srid
 system         pg_extension  spatial_ref_sys                  srtext                                                                                                    4
 system         public        sql_instances                    addr                                                                                                      2
 system         public        sql_instances                    id                                                                                                        1
+system         public        sql_instances                    locality                                                                                                  4
 system         public        sql_instances                    session_id                                                                                                3
 system         public        sqlliveness                      expiration                                                                                                2
 system         public        sqlliveness                      session_id                                                                                                1

pkg/sql/sqlinstance/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/base",
+        "//pkg/roachpb",
         "//pkg/sql/sqlliveness",
         "@com_github_cockroachdb_errors//:errors",
     ],

pkg/sql/sqlinstance/instanceprovider/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "//pkg/keys",
         "//pkg/kv",
         "//pkg/kv/kvclient/rangefeed",
+        "//pkg/roachpb",
         "//pkg/sql/sqlinstance",
         "//pkg/sql/sqlinstance/instancestorage",
         "//pkg/sql/sqlliveness",

pkg/sql/sqlinstance/instanceprovider/instanceprovider.go

@@ -19,6 +19,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/kv"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlinstance"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlliveness"
@@ -31,7 +32,7 @@ import (
 )
 
 type writer interface {
-	CreateInstance(ctx context.Context, sessionID sqlliveness.SessionID, sessionExpiration hlc.Timestamp, instanceAddr string) (base.SQLInstanceID, error)
+	CreateInstance(ctx context.Context, sessionID sqlliveness.SessionID, sessionExpiration hlc.Timestamp, instanceAddr string, locality roachpb.Locality) (base.SQLInstanceID, error)
 	ReleaseInstanceID(ctx context.Context, instanceID base.SQLInstanceID) error
 }
 
@@ -42,6 +43,7 @@ type provider struct {
 	stopper      *stop.Stopper
 	instanceAddr string
 	session      sqlliveness.Instance
+	locality     roachpb.Locality
 	initOnce     sync.Once
 	initialized  chan struct{}
 	instanceID   base.SQLInstanceID
@@ -60,6 +62,7 @@ func New(
 	codec keys.SQLCodec,
 	slProvider sqlliveness.Provider,
 	addr string,
+	locality roachpb.Locality,
 	f *rangefeed.Factory,
 	clock *hlc.Clock,
 ) sqlinstance.Provider {
@@ -71,6 +74,7 @@ func New(
 		Reader:       reader,
 		session:      slProvider,
 		instanceAddr: addr,
+		locality:     locality,
 		initialized:  make(chan struct{}),
 	}
 	return p
@@ -142,7 +146,7 @@ func (p *provider) initialize(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "constructing session")
 	}
-	instanceID, err := p.storage.CreateInstance(ctx, session.ID(), session.Expiration(), p.instanceAddr)
+	instanceID, err := p.storage.CreateInstance(ctx, session.ID(), session.Expiration(), p.instanceAddr, p.locality)
 	if err != nil {
 		return err
 	}

pkg/sql/sqlinstance/instancestorage/BUILD.bazel

@@ -29,6 +29,7 @@ go_library(
         "//pkg/util/encoding",
         "//pkg/util/grpcutil",
         "//pkg/util/hlc",
+        "//pkg/util/json",
         "//pkg/util/log",
         "//pkg/util/stop",
         "//pkg/util/syncutil",
@@ -48,6 +49,7 @@ go_test(
         "//pkg/base",
         "//pkg/keys",
         "//pkg/kv/kvclient/rangefeed",
+        "//pkg/roachpb",
         "//pkg/security/securityassets",
         "//pkg/security/securitytest",
         "//pkg/server",

pkg/sql/sqlinstance/instancestorage/instancereader.go

@@ -118,7 +118,7 @@ func (r *Reader) maybeStartRangeFeed(ctx context.Context) *rangefeed.RangeFeed {
 	updateCacheFn := func(
 		ctx context.Context, keyVal *roachpb.RangeFeedValue,
 	) {
-		instanceID, addr, sessionID, timestamp, tombstone, err := r.rowcodec.decodeRow(kv.KeyValue{
+		instanceID, addr, sessionID, locality, timestamp, tombstone, err := r.rowcodec.decodeRow(kv.KeyValue{
 			Key:   keyVal.Key,
 			Value: &keyVal.Value,
 		})
@@ -131,6 +131,7 @@ func (r *Reader) maybeStartRangeFeed(ctx context.Context) *rangefeed.RangeFeed {
 			addr:       addr,
 			sessionID:  sessionID,
 			timestamp:  timestamp,
+			locality:   locality,
 		}
 		r.updateInstanceMap(instance, tombstone)
 	}
@@ -195,6 +196,7 @@ func (r *Reader) GetInstance(
 		InstanceID:   instance.instanceID,
 		InstanceAddr: instance.addr,
 		SessionID:    instance.sessionID,
+		Locality:     instance.locality,
 	}
 	return instanceInfo, nil
 }
@@ -218,6 +220,7 @@ func (r *Reader) GetAllInstances(
 			InstanceID:   liveInstance.instanceID,
 			InstanceAddr: liveInstance.addr,
 			SessionID:    liveInstance.sessionID,
+			Locality:     liveInstance.locality,
 		}
 		sqlInstances = append(sqlInstances, instanceInfo)
 	}