roachpb: InternalServer API for tenant settings

This commit introduces the API that the tenants will use to obtain and
list for updates to tenant setting overrides. The API was designed to
allow for maximum flexibility on the server side so that it can be
implemented as a range feed without any extra state (if necessary).

Release note: None

pkg/ccl/kvccl/kvtenantccl/connector_test.go

@@ -100,6 +100,12 @@ func (m *mockServer) UpdateSpanConfigs(
 	panic("unimplemented")
 }
 
+func (m *mockServer) TenantSettings(
+	*roachpb.TenantSettingsRequest, roachpb.Internal_TenantSettingsServer,
+) error {
+	panic("unimplemented")
+}
+
 func gossipEventForClusterID(clusterID uuid.UUID) *roachpb.GossipSubscriptionEvent {
 	return &roachpb.GossipSubscriptionEvent{
 		Key:            gossip.KeyClusterID,

pkg/kv/kvclient/kvcoord/send_test.go

@@ -91,6 +91,12 @@ func (n Node) UpdateSpanConfigs(
 	panic("unimplemented")
 }
 
+func (n Node) TenantSettings(
+	*roachpb.TenantSettingsRequest, roachpb.Internal_TenantSettingsServer,
+) error {
+	panic("unimplemented")
+}
+
 // TestSendToOneClient verifies that Send correctly sends a request
 // to one server using the heartbeat RPC.
 func TestSendToOneClient(t *testing.T) {

pkg/kv/kvclient/kvcoord/transport_test.go

@@ -208,3 +208,9 @@ func (m *mockInternalClient) UpdateSpanConfigs(
 ) (*roachpb.UpdateSpanConfigsResponse, error) {
 	return nil, fmt.Errorf("unsupported UpdateSpanConfigs call")
 }
+
+func (m *mockInternalClient) TenantSettings(
+	*roachpb.TenantSettingsRequest, roachpb.Internal_TenantSettingsServer,
+) error {
+	return nil, fmt.Errorf("unsupported TenantSettings call")
+}

pkg/roachpb/api.go

@@ -1677,3 +1677,19 @@ func (s *ScanStats) SafeFormat(w redact.SafePrinter, _ rune) {
 func (s *ScanStats) String() string {
 	return redact.StringWithoutMarkers(s)
 }
+
+// TenantSettingsPrecedence identifies the precedence of a set of setting
+// overrides. It is used by the TenantSettings API which supports passing
+// multiple overrides for the same setting.
+type TenantSettingsPrecedence uint32
+
+const (
+	// SpecificTenantOverrides is the high precedence for tenant setting overrides.
+	// These overrides take precedence over AllTenantsOverrides.
+	SpecificTenantOverrides TenantSettingsPrecedence = 1 + iota
+
+	// AllTenantsOverrides is the low precedence for tenant setting overrides.
+	// These overrides are only effectual for a tenant if there is no override
+	// with the SpecificTenantOverrides precedence..
+	AllTenantsOverrides
+)

pkg/roachpb/api.proto

@@ -2610,6 +2610,60 @@ message GossipSubscriptionEvent {
   kv.kvpb.Error error = 4;
 }
 
+// TenantSettingsRequest establishes an indefinite stream that provides
+// up-to-date overrides for tenant settings.
+//
+// Upon establishment of the stream, the current overrides are returned as an
+// event, and any time the overrides change a new event is generated.
+message TenantSettingsRequest {
+  TenantID tenant_id = 1 [(gogoproto.customname) = "TenantID", (gogoproto.nullable) = false];
+}
+
+// TenantSettingsEvent is used to report changes to setting overrides for a
+// particular tenant.
+//
+// Each event pertains to a certain precedence value (see
+// TenantSettingsPrecedence).
+//
+// Note: this API is designed to allow flexibility of implementation on the
+// server side (e.g. to make it maintain very little state per tenant).
+message TenantSettingsEvent {
+  // Precedence must be a valid TenantSettingsPrecedence value.
+  uint32 precedence = 1 [(gogoproto.casttype) = "TenantSettingsPrecedence"];
+
+  // Incremental is true if the list of overrides is a list of changes since the
+  // last event. In that case, any overrides that have been removed are returned
+  // as TenantSettings with empty RawValue and ValueType fields.
+  //
+  // When Incremental is false, the overrides contains the complete list of
+  // current overrides for this precedence.
+  //
+  // The first event for a precedence is never incremental.
+  bool incremental = 2;
+
+  // Overrides contains:
+  //  - all current setting overrides for the given precedence if Incremental is
+  //    false; or
+  //  - the changed overrides since the last event for the precedence if
+  //    Incremental is true (removed overrides have empty RawValue and ValueType
+  //    fields).
+  repeated TenantSetting overrides = 3;
+
+  // If non-nil, the other fields will be empty and this will be the final event
+  // sent on the stream before it is terminated.
+  errorspb.EncodedError error = 4 [(gogoproto.nullable) = false];
+}
+
+// TenantSetting contains the name and value of a tenant setting.
+//
+// The value representation is the same as that used by the system.settings
+// table (value and valueType columns).
+message TenantSetting {
+  string name = 1;
+  string raw_value = 2;
+  string value_type = 3;
+}
+
 // TenantConsumption contains information about resource utilization by a
 // tenant, which directly factors into their bill.
 message TenantConsumption {
@@ -2743,6 +2797,10 @@ service Internal {
   // UpdateSpanConfigs is used to update the span configurations over given
   // keyspans.
   rpc UpdateSpanConfigs (UpdateSpanConfigsRequest) returns (UpdateSpanConfigsResponse) { }
+
+  // TenantSettings is used by tenants to obtain and stay up to date with tenant
+  // setting overrides.
+  rpc TenantSettings (TenantSettingsRequest) returns (stream TenantSettingsEvent) { }
 }
 
 // ContentionEvent is a message that will be attached to BatchResponses

pkg/rpc/context.go

@@ -743,6 +743,48 @@ func (a internalClientAdapter) GossipSubscription(
 	return gsAdapter, nil
 }
 
+type tenantSettingsClientAdapter struct {
+	respStreamClientAdapter
+}
+
+// roachpb.Internal_TenantSettingsServer methods.
+func (a tenantSettingsClientAdapter) Recv() (*roachpb.TenantSettingsEvent, error) {
+	e, err := a.recvInternal()
+	if err != nil {
+		return nil, err
+	}
+	return e.(*roachpb.TenantSettingsEvent), nil
+}
+
+// roachpb.Internal_TenantSettingsServer methods.
+func (a tenantSettingsClientAdapter) Send(e *roachpb.TenantSettingsEvent) error {
+	return a.sendInternal(e)
+}
+
+var _ roachpb.Internal_TenantSettingsClient = tenantSettingsClientAdapter{}
+var _ roachpb.Internal_TenantSettingsServer = tenantSettingsClientAdapter{}
+
+// TenantSettings is part of the roachpb.InternalClient interface.
+func (a internalClientAdapter) TenantSettings(
+	ctx context.Context, args *roachpb.TenantSettingsRequest, _ ...grpc.CallOption,
+) (roachpb.Internal_TenantSettingsClient, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	gsAdapter := tenantSettingsClientAdapter{
+		respStreamClientAdapter: makeRespStreamClientAdapter(ctx),
+	}
+
+	go func() {
+		defer cancel()
+		err := a.server.TenantSettings(args, gsAdapter)
+		if err == nil {
+			err = io.EOF
+		}
+		gsAdapter.errC <- err
+	}()
+
+	return gsAdapter, nil
+}
+
 var _ roachpb.InternalClient = internalClientAdapter{}
 
 // IsLocal returns true if the given InternalClient is local.

pkg/rpc/context_test.go

@@ -276,6 +276,12 @@ func (*internalServer) UpdateSpanConfigs(
 	panic("unimplemented")
 }
 
+func (*internalServer) TenantSettings(
+	*roachpb.TenantSettingsRequest, roachpb.Internal_TenantSettingsServer,
+) error {
+	panic("unimplemented")
+}
+
 // TestInternalServerAddress verifies that RPCContext uses AdvertiseAddr, not Addr, to
 // determine whether to apply the local server optimization.
 //

pkg/rpc/nodedialer/nodedialer_test.go

@@ -597,3 +597,9 @@ func (*internalServer) UpdateSpanConfigs(
 ) (*roachpb.UpdateSpanConfigsResponse, error) {
 	panic("unimplemented")
 }
+
+func (*internalServer) TenantSettings(
+	*roachpb.TenantSettingsRequest, roachpb.Internal_TenantSettingsServer,
+) error {
+	panic("unimplemented")
+}

pkg/server/node.go

@@ -1375,6 +1375,13 @@ func (n *Node) GossipSubscription(
 	}
 }
 
+// TenantSettings implements the roachpb.InternalServer interface.
+func (n *Node) TenantSettings(
+	args *roachpb.TenantSettingsRequest, stream roachpb.Internal_TenantSettingsServer,
+) error {
+	return errors.AssertionFailedf("not implemented")
+}
+
 // Join implements the roachpb.InternalServer service. This is the
 // "connectivity" API; individual CRDB servers are passed in a --join list and
 // the join targets are addressed through this API.