From 00552aa4f9733974ac104b5543ae7486aa8fd1cb Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Mon, 27 Mar 2023 15:29:04 +0000 Subject: [PATCH 1/4] spanconfigbounds: fix bug in how Int32 ranges are clamped Previously, when constructing ValueBounds for any Int32 fields, we would check if the ConstraintBounds field was non-null, which wasn't intentional. This meant that we wouldn't clamp fields like GC TTL, number of voters, and number of replicas if ConstraintBounds wasn't set. Epic: none Release note: None --- pkg/spanconfig/spanconfigbounds/int32field.go | 3 --- .../spanconfigbounds/testdata/basic | 20 ++++++++++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/spanconfig/spanconfigbounds/int32field.go b/pkg/spanconfig/spanconfigbounds/int32field.go index d0310539fdd0..e35f30c0a61c 100644 --- a/pkg/spanconfig/spanconfigbounds/int32field.go +++ b/pkg/spanconfig/spanconfigbounds/int32field.go @@ -30,9 +30,6 @@ func (f int32Field) SafeFormat(s redact.SafePrinter, verb rune) { func (f int32Field) FieldBound(b Bounds) ValueBounds { getBound := func() *tenantcapabilitiespb.SpanConfigBounds_Int32Range { - if b.b.ConstraintBounds == nil { - return nil - } switch f { case numReplicas: return b.b.NumReplicas diff --git a/pkg/spanconfig/spanconfigbounds/testdata/basic b/pkg/spanconfig/spanconfigbounds/testdata/basic index c3a52ad58d89..f89ab542d16b 100644 --- a/pkg/spanconfig/spanconfigbounds/testdata/basic +++ b/pkg/spanconfig/spanconfigbounds/testdata/basic @@ -1,12 +1,16 @@ bounds name=foo gc_ttl_seconds: range_max_bytes: +num_voters: +num_replicas: ---- config name=bar gc_policy: range_min_bytes: 5 range_max_bytes: 5 +num_voters: 7 +num_replicas: 7 ---- conforms bounds=foo config=bar @@ -15,23 +19,29 @@ false check bounds=foo config=bar ---- -span config bounds violated for fields: range_max_bytes -span config bounds violated for fields: range_max_bytes -(1) span config bounds violated for fields: range_max_bytes +span config bounds violated for fields: range_max_bytes, num_voters, gc.ttlseconds +span config bounds violated for fields: range_max_bytes, num_voters, gc.ttlseconds +(1) span config bounds violated for fields: range_max_bytes, num_voters, gc.ttlseconds | range_max_bytes: 5 does not conform to [10, 20], will be clamped to 10 + | num_voters: 7 does not conform to [3, 5], will be clamped to 5 + | gc.ttlseconds: 122 does not conform to [123, 7000], will be clamped to 123 Error types: (1) *spanconfigbounds.ViolationError clamp bounds=foo config=bar ---- ---- -@@ -1,6 +1,6 @@ +@@ -1,8 +1,8 @@ range_min_bytes: 5 -range_max_bytes: 5 +range_max_bytes: 10 gc_policy: < - ttl_seconds: 122 +- ttl_seconds: 122 ++ ttl_seconds: 123 > + num_replicas: 7 +-num_voters: 7 ++num_voters: 5 ---- ---- From 72d207b735ac052791e054b1932b429697f4d8fd Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Mon, 27 Mar 2023 16:29:10 +0000 Subject: [PATCH 2/4] spanconfigtestutils: add ability to declare tenant keys/spans This patch allows us to declare keys/spans as datadriven input for datadriven tests in the spanconfig package. We'll make use of this in an upcoming commit. Epic: none Release note: None --- .../spanconfigkvsubscriber/datadriven_test.go | 1 - .../spanconfigtestutils/BUILD.bazel | 1 + pkg/spanconfig/spanconfigtestutils/utils.go | 160 ++++++++++++++++-- .../spanconfigtestutils/utils_test.go | 40 +++-- 4 files changed, 179 insertions(+), 23 deletions(-) diff --git a/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go b/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go index d3383bf71a4b..ef4b3b653734 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go +++ b/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go @@ -100,7 +100,6 @@ import ( // Text of the form [a,b) and [a,b):C correspond to spans and span config // records; see spanconfigtestutils.Parse{Span,Config,SpanConfigRecord} for more // details. -// TODO(arul): Add ability to express tenant spans to this datadriven test. func TestDataDriven(t *testing.T) { defer leaktest.AfterTest(t)() diff --git a/pkg/spanconfig/spanconfigtestutils/BUILD.bazel b/pkg/spanconfig/spanconfigtestutils/BUILD.bazel index ccd602dbb69f..2b9c5048a14f 100644 --- a/pkg/spanconfig/spanconfigtestutils/BUILD.bazel +++ b/pkg/spanconfig/spanconfigtestutils/BUILD.bazel @@ -11,6 +11,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/config/zonepb", + "//pkg/keys", "//pkg/kv", "//pkg/kv/kvserver/protectedts/ptpb", "//pkg/roachpb", diff --git a/pkg/spanconfig/spanconfigtestutils/utils.go b/pkg/spanconfig/spanconfigtestutils/utils.go index 527592bac371..771b66d5512b 100644 --- a/pkg/spanconfig/spanconfigtestutils/utils.go +++ b/pkg/spanconfig/spanconfigtestutils/utils.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/config/zonepb" + "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptpb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/spanconfig" @@ -33,7 +34,7 @@ import ( // spanRe matches strings of the form "[start, end)", capturing both the "start" // and "end" keys. -var spanRe = regexp.MustCompile(`^\[(\w+),\s??(\w+)\)$`) +var spanRe = regexp.MustCompile(`^\[((/Tenant/\d*/)?\w+),\s??((/Tenant/\d*/)?\w+)\)$`) // systemTargetRe matches strings of the form // "{entire-keyspace|source=,(target=|all-tenant-keyspace-targets-set)}". @@ -45,6 +46,14 @@ var systemTargetRe = regexp.MustCompile( // shorthand for declaring a unique tagged config. var configRe = regexp.MustCompile(`^(FALLBACK)|(^\w)$`) +// tenantRe matches a string of the form "/Tenant/". +var tenantRe = regexp.MustCompile(`/Tenant/(\d*)`) + +// keyRe matches a string of the form "a", "Tenant/10/a". An optional tenant +// prefix may be used to specify a key inside a secondary tenant's keyspace; +// otherwise, the key is within the system tenant's keyspace. +var keyRe = regexp.MustCompile(`(Tenant/\d*/)?(\w+)`) + // ParseRangeID is helper function that constructs a roachpb.RangeID from a // string of the form "r". func ParseRangeID(t testing.TB, s string) roachpb.RangeID { @@ -145,13 +154,50 @@ func ParseSpan(t testing.TB, sp string) roachpb.Span { } matches := spanRe.FindStringSubmatch(sp) - start, end := matches[1], matches[2] + startStr, endStr := matches[1], matches[3] + start, tenStart := ParseKey(t, startStr) + end, tenEnd := ParseKey(t, endStr) + + // Sanity check keys don't straddle tenant boundaries. + require.Equal(t, tenStart, tenEnd) + return roachpb.Span{ - Key: roachpb.Key(start), - EndKey: roachpb.Key(end), + Key: start, + EndKey: end, } } +// ParseKey constructs a roachpb.Key from the supplied input. The key may be +// optionally prefixed with a "/Tenant/ID/" prefix; doing so ensures the key +// belongs to the specified tenant's keyspace. Otherwise, the key lies in the +// system tenant's keyspace. +// +// In addition to the key, the tenant ID is also returned. +func ParseKey(t testing.TB, key string) (roachpb.Key, roachpb.TenantID) { + require.True(t, keyRe.MatchString(key)) + + matches := keyRe.FindStringSubmatch(key) + tenantID := roachpb.SystemTenantID + if matches[1] != "" { + tenantID = parseTenant(t, key) + } + + codec := keys.MakeSQLCodec(tenantID) + return append(codec.TenantPrefix(), roachpb.Key(matches[2])...), tenantID +} + +// parseTenant expects a string of the form "ten-" and returns the +// tenant ID. +func parseTenant(t testing.TB, input string) roachpb.TenantID { + require.True(t, tenantRe.MatchString(input), input) + + matches := tenantRe.FindStringSubmatch(input) + tenID := matches[1] + tenIDRaw, err := strconv.Atoi(tenID) + require.NoError(t, err) + return roachpb.MustMakeTenantID(uint64(tenIDRaw)) +} + // parseSystemTarget is a helepr function that constructs a // spanconfig.SystemTarget from a string of the form {source=,target=} func parseSystemTarget(t testing.TB, systemTarget string) spanconfig.SystemTarget { @@ -220,6 +266,73 @@ func ParseConfig(t testing.TB, conf string) roachpb.SpanConfig { } } +// ParseDeclareBoundsArguments parses datadriven test input to a list of +// tenantcapabilities.Updates that can be applied to a +// tenantcapabilities.Reader. The input is of the following form: +// +// delete ten-10 +// set ten-10:{GC.ttl_start=5, GC.ttl_end=30} +func ParseDeclareBoundsArguments(t *testing.T, input string) []tenantcapabilities.Update { + var updates []tenantcapabilities.Update + for _, line := range strings.Split(input, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + const setPrefix, deletePrefix = "set ", "delete " + + switch { + case strings.HasPrefix(line, setPrefix): + line = strings.TrimPrefix(line, line[:len(setPrefix)]) + parts := strings.Split(line, ":") + require.Len(t, parts, 2) + tenantID := parseTenant(t, parts[0]) + bounds := parseBounds(t, parts[1]) + updates = append(updates, tenantcapabilities.Update{ + Entry: tenantcapabilities.Entry{ + TenantID: tenantID, + TenantCapabilities: &tenantcapabilitiespb.TenantCapabilities{ + SpanConfigBounds: bounds, + }, + }, + }) + case strings.HasPrefix(line, deletePrefix): + line = strings.TrimPrefix(line, line[:len(deletePrefix)]) + tenantID := parseTenant(t, line) + updates = append(updates, tenantcapabilities.Update{ + Entry: tenantcapabilities.Entry{ + TenantID: tenantID, + }, + Deleted: true, + }) + default: + t.Fatalf("malformed line %q, expected to find prefix %q or %q", + line, setPrefix, deletePrefix) + } + } + return updates +} + +// parseBounds parses a string that looks like {GC.ttl_start=5, GC.ttl_end=40} +// into a SpanConfigBoudns struct. +func parseBounds(t *testing.T, input string) *tenantcapabilitiespb.SpanConfigBounds { + require.True(t, boundsRe.MatchString(input)) + + matches := boundsRe.FindStringSubmatch(input) + gcTTLStart, err := strconv.Atoi(matches[1]) + require.NoError(t, err) + gcTTLEnd, err := strconv.Atoi(matches[2]) + require.NoError(t, err) + + return &tenantcapabilitiespb.SpanConfigBounds{ + GCTTLSeconds: &tenantcapabilitiespb.SpanConfigBounds_Int32Range{ + Start: int32(gcTTLStart), + End: int32(gcTTLEnd), + }, + } +} + // ParseSpanConfigRecord is helper function that constructs a // spanconfig.Target from a string of the form target:config. See // ParseTarget and ParseConfig above. @@ -347,21 +460,44 @@ func ParseStoreApplyArguments(t testing.TB, input string) (updates []spanconfig. // roundtrip; spans containing special keys that translate to pretty-printed // keys are printed as such. func PrintSpan(sp roachpb.Span) string { - s := []string{ - sp.Key.String(), - sp.EndKey.String(), - } - for i := range s { + var res []string + for _, key := range []roachpb.Key{sp.Key, sp.EndKey} { + var tenantPrefixStr string + rest, tenID, err := keys.DecodeTenantPrefix(key) + if err != nil { + panic(err) + } + + if !tenID.IsSystem() { + tenantPrefixStr = fmt.Sprintf("%s", keys.MakeSQLCodec(tenID).TenantPrefix()) + } + // Raw keys are quoted, so we unquote them. - if strings.Contains(s[i], "\"") { + restStr := roachpb.Key(rest).String() + if strings.Contains(restStr, "\"") { var err error - s[i], err = strconv.Unquote(s[i]) + restStr, err = strconv.Unquote(restStr) if err != nil { panic(err) } } + + // For keys inside a secondary tenant, we don't print the "/Min" suffix if + // the key is at the start of their keyspace. Also, we add a "/" delimiter + // after the tenant prefix. + if !tenID.IsSystem() { + if roachpb.Key(rest).Compare(keys.MinKey) == 0 { + restStr = "" + } + + if restStr != "" { + restStr = fmt.Sprintf("/%s", restStr) + } + } + + res = append(res, fmt.Sprintf("%s%s", tenantPrefixStr, restStr)) } - return fmt.Sprintf("[%s,%s)", s[0], s[1]) + return fmt.Sprintf("[%s,%s)", res[0], res[1]) } // PrintTarget is a helper function that prints a spanconfig.Target. diff --git a/pkg/spanconfig/spanconfigtestutils/utils_test.go b/pkg/spanconfig/spanconfigtestutils/utils_test.go index 1df3189d16f3..3c3c5ed54295 100644 --- a/pkg/spanconfig/spanconfigtestutils/utils_test.go +++ b/pkg/spanconfig/spanconfigtestutils/utils_test.go @@ -11,6 +11,7 @@ package spanconfigtestutils import ( + "strings" "testing" "github.com/stretchr/testify/require" @@ -20,16 +21,26 @@ func TestSpanRe(t *testing.T) { for _, tc := range []struct { input string expMatch bool + expTenant bool expStart, expEnd string }{ - {"[a, b)", true, "a", "b"}, - {"[acd, bfg)", true, "acd", "bfg"}, // multi character keys allowed - {"[a,b)", true, "a", "b"}, // separating space is optional - {"[ a,b) ", false, "", ""}, // extraneous spaces disallowed - {"[a,b ) ", false, "", ""}, // extraneous spaces disallowed - {"[a,, b)", false, "", ""}, // only single comma allowed - {" [a, b)", false, "", ""}, // need to start with '[' - {"[a,b)x", false, "", ""}, // need to end with ')' + {"[a, b)", true, false, "a", "b"}, + // Multi character keys allowed. + {"[acd, bfg)", true, false, "acd", "bfg"}, + // Separating space is optional. + {"[a,b)", true, false, "a", "b"}, + // Tenant span. + {"[/Tenant/10/a,/Tenant/10/b)", true, true, "/Tenant/10/a", "/Tenant/10/b"}, + // Extraneous spaces disallowed. + {"[ a,b) ", false, false, "", ""}, + // Extraneous spaces disallowed. + {"[a,b ) ", false, false, "", ""}, + // Only single comma allowed. + {"[a,, b)", false, false, "", ""}, + // Need to start with '['. + {" [a, b)", false, false, "", ""}, + // Need to end with ')'. + {"[a,b)x", false, false, "", ""}, } { require.Equalf(t, tc.expMatch, spanRe.MatchString(tc.input), "input = %s", tc.input) if !tc.expMatch { @@ -37,8 +48,17 @@ func TestSpanRe(t *testing.T) { } matches := spanRe.FindStringSubmatch(tc.input) - require.Len(t, matches, 3) - start, end := matches[1], matches[2] + require.Len(t, matches, 5) + start, end := matches[1], matches[3] + + const tenPrefix = "/Tenant/" + if tc.expTenant { + require.True(t, strings.HasPrefix(matches[2], tenPrefix)) + require.True(t, strings.HasPrefix(matches[4], tenPrefix)) + } else { + require.Equal(t, 0, len(matches[2])) + require.Equal(t, 0, len(matches[4])) + } require.Equal(t, tc.expStart, start) require.Equal(t, tc.expEnd, end) } From 0664f06efd8642b00f7c44da3f80de1cb85473bf Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Mon, 27 Mar 2023 17:09:49 +0000 Subject: [PATCH 3/4] spanconfig: integrate SpanConfigBounds with the Store and KVSubscriber This patch integrates `SpanConfigBounds` with the `KVSubscriber` and `spanconfig.Store`. The `spanconfig.Store` used by the `KVSubscriber` now has a handle to the global tenant capability state. It uses this to clamp any secondary tenant span configs that are not in conformance before returning them. By default, clamping of secondary tenant span configurations is turned off. It can be enabled using the `spanconfig.bounds.enabled` cluster setting. The setting is hidden. Fixes: #99689 Release note: None --- pkg/kv/kvserver/BUILD.bazel | 1 + pkg/kv/kvserver/client_spanconfigs_test.go | 6 +- .../tenantcapabilities/capabilities.go | 18 +++++ .../tenantcapabilitiespb/capabilities.go | 30 +++++++- pkg/server/server.go | 19 ++--- .../spanconfigbounds/span_config_bounds.go | 2 +- .../spanconfigkvsubscriber/BUILD.bazel | 2 + .../spanconfigkvsubscriber/datadriven_test.go | 2 + .../spanconfigkvsubscriber/kvsubscriber.go | 18 +++-- .../kvsubscriber_test.go | 2 + .../spanconfigreconciler/BUILD.bazel | 1 + .../spanconfigreconciler/reconciler.go | 5 +- pkg/spanconfig/spanconfigreporter/BUILD.bazel | 1 + .../spanconfigreporter/datadriven_test.go | 5 +- pkg/spanconfig/spanconfigstore/BUILD.bazel | 4 ++ pkg/spanconfig/spanconfigstore/store.go | 67 +++++++++++++++-- pkg/spanconfig/spanconfigstore/store_test.go | 61 +++++++++++++++- .../spanconfigstore/testdata/bounds/basic | 72 +++++++++++++++++++ .../spanconfigtestutils/BUILD.bazel | 2 + pkg/spanconfig/spanconfigtestutils/utils.go | 48 +++++++++---- pkg/testutils/localtestcluster/BUILD.bazel | 1 + .../localtestcluster/local_test_cluster.go | 6 +- 22 files changed, 329 insertions(+), 44 deletions(-) create mode 100644 pkg/spanconfig/spanconfigstore/testdata/bounds/basic diff --git a/pkg/kv/kvserver/BUILD.bazel b/pkg/kv/kvserver/BUILD.bazel index c244d53fe9e2..9e6f0ec2bab2 100644 --- a/pkg/kv/kvserver/BUILD.bazel +++ b/pkg/kv/kvserver/BUILD.bazel @@ -389,6 +389,7 @@ go_test( "//pkg/kv/kvserver/tscache", "//pkg/kv/kvserver/txnwait", "//pkg/kv/kvserver/uncertainty", + "//pkg/multitenant/tenantcapabilities", "//pkg/multitenant/tenantcapabilities/tenantcapabilitiesauthorizer", "//pkg/roachpb", "//pkg/rpc", diff --git a/pkg/kv/kvserver/client_spanconfigs_test.go b/pkg/kv/kvserver/client_spanconfigs_test.go index 4c0e9d5e73c3..115704076ea0 100644 --- a/pkg/kv/kvserver/client_spanconfigs_test.go +++ b/pkg/kv/kvserver/client_spanconfigs_test.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" @@ -41,6 +42,7 @@ func TestSpanConfigUpdateAppliedToReplica(t *testing.T) { spanConfigStore := spanconfigstore.New( roachpb.TestingDefaultSpanConfig(), cluster.MakeTestingClusterSettings(), + tenantcapabilities.NewEmptyReader(), nil, ) var t0 = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) @@ -107,7 +109,9 @@ func TestFallbackSpanConfigOverride(t *testing.T) { defer leaktest.AfterTest(t)() st := cluster.MakeTestingClusterSettings() - spanConfigStore := spanconfigstore.New(roachpb.TestingDefaultSpanConfig(), st, nil) + spanConfigStore := spanconfigstore.New( + roachpb.TestingDefaultSpanConfig(), st, tenantcapabilities.NewEmptyReader(), nil, + ) var t0 = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) mockSubscriber := newMockSpanConfigSubscriber(t0, spanConfigStore) diff --git a/pkg/multitenant/tenantcapabilities/capabilities.go b/pkg/multitenant/tenantcapabilities/capabilities.go index 2a7c0056ddaa..528573b67e43 100644 --- a/pkg/multitenant/tenantcapabilities/capabilities.go +++ b/pkg/multitenant/tenantcapabilities/capabilities.go @@ -249,3 +249,21 @@ func (c CapabilityID) CapabilityType() Type { panic(errors.AssertionFailedf("missing case: %q", c)) } } + +// NewEmptyReader returns a new Reader which corresponds to an empty global +// capability state. +func NewEmptyReader() Reader { + return emptyReader(true) +} + +type emptyReader bool + +// GetCapabilities implements the Reader interface. +func (emptyReader) GetCapabilities(roachpb.TenantID) (TenantCapabilities, bool) { + return nil, false +} + +// GetGlobalCapabilityState implements the Reader interface. +func (emptyReader) GetGlobalCapabilityState() map[roachpb.TenantID]TenantCapabilities { + return nil +} diff --git a/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb/capabilities.go b/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb/capabilities.go index d82dd5682052..4eda2d707a15 100644 --- a/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb/capabilities.go +++ b/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb/capabilities.go @@ -29,9 +29,15 @@ func (b boolCapValue) SafeFormat(p redact.SafePrinter, verb rune) { p.Print(redact.Safe(bool(b))) } -// Unwrap implements the tenantcapabilities.Value interface. func (b boolCapValue) Unwrap() interface{} { return bool(b) } +// Unwrap implements the tenantcapabilities.Value interface. +func (m *SpanConfigBounds) Unwrap() interface{} { return m } + +func (m *SpanConfigBounds) SafeFormat(p redact.SafePrinter, verb rune) { + p.Print(redact.SafeString(m.String())) +} + // boolCap is an accessor struct for boolean capabilities. type boolCap struct { cap *bool @@ -73,6 +79,26 @@ func (i invertedBoolCap) Set(val interface{}) { *i.cap = !bval } +// spanConfigBoundsCap is an accessor struct for SpanConfigBounds that are +// stored on the underlying TenantCapabilities proto. +type spanConfigBoundsCap struct { + cap *SpanConfigBounds +} + +// Get implements the tenantcapabilities.Capability interface. +func (s spanConfigBoundsCap) Get() tenantcapabilities.Value { + return s.cap +} + +// Set implements the tenantcapabilities.Capability interface. +func (s spanConfigBoundsCap) Set(val interface{}) { + scfgBoundsVal, ok := val.(SpanConfigBounds) + if !ok { + panic(errors.AssertionFailedf("invalid value type: %T", val)) + } + *s.cap = scfgBoundsVal +} + // Cap implements the tenantcapabilities.TenantCapabilities interface. func (t *TenantCapabilities) Cap( capabilityID tenantcapabilities.CapabilityID, @@ -92,6 +118,8 @@ func (t *TenantCapabilities) Cap( return boolCap{&t.CanViewTSDBMetrics} case tenantcapabilities.ExemptFromRateLimiting: return boolCap{&t.ExemptFromRateLimiting} + case tenantcapabilities.TenantSpanConfigBounds: + return spanConfigBoundsCap{t.SpanConfigBounds} default: panic(errors.AssertionFailedf("unknown capability: %q", capabilityID.String())) diff --git a/pkg/server/server.go b/pkg/server/server.go index d3ad9fcc9e98..48b8d359b63b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -623,6 +623,15 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { keys.SystemSQLCodec, clock, rangeFeedFactory, &cfg.DefaultZoneConfig, ) + tenantCapabilitiesWatcher := tenantcapabilitieswatcher.New( + clock, + rangeFeedFactory, + keys.TenantsTableID, + stopper, + 1<<20, /* 1 MB */ + tenantCapabilitiesTestingKnobs, + ) + var spanConfig struct { // kvAccessor powers the span configuration RPCs and the host tenant's // reconciliation job. @@ -668,6 +677,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { 1<<20, /* 1 MB */ fallbackConf, cfg.Settings, + tenantCapabilitiesWatcher, spanConfigKnobs, registry, ) @@ -791,15 +801,6 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { clock, rangeFeedFactory, stopper, st, ) - tenantCapabilitiesWatcher := tenantcapabilitieswatcher.New( - clock, - rangeFeedFactory, - keys.TenantsTableID, - stopper, - 1<<20, /* 1 MB */ - tenantCapabilitiesTestingKnobs, - ) - node := NewNode( storeCfg, recorder, diff --git a/pkg/spanconfig/spanconfigbounds/span_config_bounds.go b/pkg/spanconfig/spanconfigbounds/span_config_bounds.go index a9cd006532cc..4415a91493eb 100644 --- a/pkg/spanconfig/spanconfigbounds/span_config_bounds.go +++ b/pkg/spanconfig/spanconfigbounds/span_config_bounds.go @@ -77,7 +77,7 @@ func (b Bounds) Check(c *roachpb.SpanConfig) Violations { func (b Bounds) clamp(c *roachpb.SpanConfig, reporter func(Field)) (changed bool) { for _, f := range fields { - if b := f.FieldBound(b); !b.clamp(c, f) { + if bb := f.FieldBound(b); !bb.clamp(c, f) { continue } changed = true diff --git a/pkg/spanconfig/spanconfigkvsubscriber/BUILD.bazel b/pkg/spanconfig/spanconfigkvsubscriber/BUILD.bazel index 4f7f7c4c6fab..cad8e24a3389 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/BUILD.bazel +++ b/pkg/spanconfig/spanconfigkvsubscriber/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//pkg/kv/kvclient/rangefeed/rangefeedbuffer", "//pkg/kv/kvclient/rangefeed/rangefeedcache", "//pkg/kv/kvpb", + "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/settings", "//pkg/settings/cluster", @@ -57,6 +58,7 @@ go_test( "//pkg/kv/kvclient/rangefeed/rangefeedbuffer", "//pkg/kv/kvclient/rangefeed/rangefeedcache", "//pkg/kv/kvserver", + "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/security/securityassets", "//pkg/security/securitytest", diff --git a/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go b/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go index ef4b3b653734..a895b51e7126 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go +++ b/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedbuffer" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/spanconfig" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvaccessor" @@ -147,6 +148,7 @@ func TestDataDriven(t *testing.T) { 10<<20, /* 10 MB */ spanconfigtestutils.ParseConfig(t, "FALLBACK"), tc.Server(0).ClusterSettings(), + tenantcapabilities.NewEmptyReader(), &spanconfig.TestingKnobs{ KVSubscriberRangeFeedKnobs: &rangefeedcache.TestingKnobs{ OnTimestampAdvance: func(ts hlc.Timestamp) { diff --git a/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go b/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go index 285f07b37622..8e62ba0d6b77 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go +++ b/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedbuffer" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -139,6 +140,9 @@ type KVSubscriber struct { clock *hlc.Clock metrics *Metrics + + // capabilitiesReader provides a handle to the global tenant capability state. + capabilitiesReader tenantcapabilities.Reader } var _ spanconfig.KVSubscriber = &KVSubscriber{} @@ -186,6 +190,7 @@ func New( bufferMemLimit int64, fallback roachpb.SpanConfig, settings *cluster.Settings, + capabilitiesReader tenantcapabilities.Reader, knobs *spanconfig.TestingKnobs, registry *metric.Registry, ) *KVSubscriber { @@ -200,12 +205,13 @@ func New( Key: spanConfigTableStart, EndKey: spanConfigTableStart.PrefixEnd(), } - spanConfigStore := spanconfigstore.New(fallback, settings, knobs) + spanConfigStore := spanconfigstore.New(fallback, settings, capabilitiesReader, knobs) s := &KVSubscriber{ - fallback: fallback, - knobs: knobs, - settings: settings, - clock: clock, + fallback: fallback, + knobs: knobs, + settings: settings, + clock: clock, + capabilitiesReader: capabilitiesReader, } var rfCacheKnobs *rangefeedcache.TestingKnobs if knobs != nil { @@ -395,7 +401,7 @@ func (s *KVSubscriber) handleUpdate(ctx context.Context, u rangefeedcache.Update func (s *KVSubscriber) handleCompleteUpdate( ctx context.Context, ts hlc.Timestamp, events []rangefeedbuffer.Event, ) { - freshStore := spanconfigstore.New(s.fallback, s.settings, s.knobs) + freshStore := spanconfigstore.New(s.fallback, s.settings, s.capabilitiesReader, s.knobs) for _, ev := range events { freshStore.Apply(ctx, false /* dryrun */, ev.(*bufferEvent).Update) } diff --git a/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber_test.go b/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber_test.go index 8bc1f7842b5f..f7b0f0f07bf3 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber_test.go +++ b/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber_test.go @@ -16,6 +16,7 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" @@ -91,6 +92,7 @@ func TestGetProtectionTimestamps(t *testing.T) { 1<<20, /* 1 MB */ roachpb.SpanConfig{}, cluster.MakeTestingClusterSettings(), + tenantcapabilities.NewEmptyReader(), nil, nil, ) diff --git a/pkg/spanconfig/spanconfigreconciler/BUILD.bazel b/pkg/spanconfig/spanconfigreconciler/BUILD.bazel index cfccd98cf5d4..3102cdf4d913 100644 --- a/pkg/spanconfig/spanconfigreconciler/BUILD.bazel +++ b/pkg/spanconfig/spanconfigreconciler/BUILD.bazel @@ -9,6 +9,7 @@ go_library( deps = [ "//pkg/keys", "//pkg/kv", + "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/settings/cluster", "//pkg/spanconfig", diff --git a/pkg/spanconfig/spanconfigreconciler/reconciler.go b/pkg/spanconfig/spanconfigreconciler/reconciler.go index 052576843e19..e1c2b8e735b3 100644 --- a/pkg/spanconfig/spanconfigreconciler/reconciler.go +++ b/pkg/spanconfig/spanconfigreconciler/reconciler.go @@ -16,6 +16,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" @@ -384,7 +385,9 @@ func (f *fullReconciler) fetchExistingSpanConfigs( targets = append(targets, spanconfig.MakeTargetFromSystemTarget(spanconfig.MakeAllTenantKeyspaceTargetsSet(f.tenID))) } - store := spanconfigstore.New(roachpb.SpanConfig{}, f.settings, f.knobs) + // The reconciler doesn't do any bounds checks or clamping, so it shouldn't + // need access to tenant capabilities (and by extension span config bounds). + store := spanconfigstore.New(roachpb.SpanConfig{}, f.settings, tenantcapabilities.NewEmptyReader(), f.knobs) { // Fully populate the store with KVAccessor contents. records, err := f.kvAccessor.GetSpanConfigRecords(ctx, targets) diff --git a/pkg/spanconfig/spanconfigreporter/BUILD.bazel b/pkg/spanconfig/spanconfigreporter/BUILD.bazel index d463555d1fb3..edf12d3efd7a 100644 --- a/pkg/spanconfig/spanconfigreporter/BUILD.bazel +++ b/pkg/spanconfig/spanconfigreporter/BUILD.bazel @@ -35,6 +35,7 @@ go_test( "//pkg/keys", "//pkg/kv/kvserver/constraint", "//pkg/kv/kvserver/liveness/livenesspb", + "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/security/securityassets", "//pkg/security/securitytest", diff --git a/pkg/spanconfig/spanconfigreporter/datadriven_test.go b/pkg/spanconfig/spanconfigreporter/datadriven_test.go index feddfd9092c5..5ca84d7001e6 100644 --- a/pkg/spanconfig/spanconfigreporter/datadriven_test.go +++ b/pkg/spanconfig/spanconfigreporter/datadriven_test.go @@ -20,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/constraint" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" clustersettings "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" @@ -233,7 +234,9 @@ func newMockCluster( nodes: make(map[roachpb.NodeID]roachpb.NodeDescriptor), ranges: make(map[roachpb.RangeID]roachpb.RangeDescriptor), liveness: make(map[roachpb.NodeID]bool), - store: spanconfigstore.New(roachpb.TestingDefaultSpanConfig(), st, scKnobs), + store: spanconfigstore.New( + roachpb.TestingDefaultSpanConfig(), st, tenantcapabilities.NewEmptyReader(), scKnobs, + ), } } diff --git a/pkg/spanconfig/spanconfigstore/BUILD.bazel b/pkg/spanconfig/spanconfigstore/BUILD.bazel index a755fa9a3093..38b774a78d94 100644 --- a/pkg/spanconfig/spanconfigstore/BUILD.bazel +++ b/pkg/spanconfig/spanconfigstore/BUILD.bazel @@ -15,10 +15,13 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/keys", + "//pkg/multitenant/tenantcapabilities", + "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb", "//pkg/roachpb", "//pkg/settings", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/util/iterutil", "//pkg/util/log", "//pkg/util/protoutil", @@ -41,6 +44,7 @@ go_test( embed = [":spanconfigstore"], deps = [ "//pkg/keys", + "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/settings/cluster", "//pkg/spanconfig", diff --git a/pkg/spanconfig/spanconfigstore/store.go b/pkg/spanconfig/spanconfigstore/store.go index 1523f6129ce8..498f79aff207 100644 --- a/pkg/spanconfig/spanconfigstore/store.go +++ b/pkg/spanconfig/spanconfigstore/store.go @@ -14,10 +14,13 @@ import ( "context" "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/errors" @@ -46,6 +49,16 @@ var FallbackConfigOverride = settings.RegisterProtobufSetting( &roachpb.SpanConfig{}, ) +// BoundsEnabled is a hidden cluster setting which controls whether +// SpanConfigBounds should be consulted (to perform clamping of secondary tenant +// span configurations) before serving span configs. +var boundsEnabled = settings.RegisterBoolSetting( + settings.SystemOnly, + "spanconfig.bounds.enabled", + "dictates whether span config bounds are consulted when serving span configs for secondary tenants", + false, +) + // Store is an in-memory data structure to store, retrieve, and incrementally // update the span configuration state. Internally, it makes use of an interval // btree based spanConfigStore to store non-overlapping span configurations that @@ -74,21 +87,28 @@ type Store struct { fallback roachpb.SpanConfig knobs *spanconfig.TestingKnobs + + // capabilitiesReader provides a handle to the global tenant capability state. + capabilitiesReader tenantcapabilities.Reader } var _ spanconfig.Store = &Store{} // New instantiates a span config store with the given fallback. func New( - fallback roachpb.SpanConfig, settings *cluster.Settings, knobs *spanconfig.TestingKnobs, + fallback roachpb.SpanConfig, + settings *cluster.Settings, + capabilitiesReader tenantcapabilities.Reader, + knobs *spanconfig.TestingKnobs, ) *Store { if knobs == nil { knobs = &spanconfig.TestingKnobs{} } s := &Store{ - settings: settings, - fallback: fallback, - knobs: knobs, + settings: settings, + fallback: fallback, + capabilitiesReader: capabilitiesReader, + knobs: knobs, } s.mu.spanConfigStore = newSpanConfigStore(settings, s.knobs) s.mu.systemSpanConfigStore = newSystemSpanConfigStore() @@ -131,7 +151,42 @@ func (s *Store) getSpanConfigForKeyRLocked( if !found { conf = s.getFallbackConfig() } - return s.mu.systemSpanConfigStore.combine(key, conf) + var err error + conf, err = s.mu.systemSpanConfigStore.combine(key, conf) + if err != nil { + return roachpb.SpanConfig{}, err + } + + // No need to perform clamping if SpanConfigBounds are not enabled. + if !boundsEnabled.Get(&s.settings.SV) { + return conf, nil + } + + _, tenID, err := keys.DecodeTenantPrefix(roachpb.Key(key)) + if err != nil { + return roachpb.SpanConfig{}, err + } + if tenID.IsSystem() { + // SpanConfig bounds do not apply to the system tenant. + return conf, nil + } + capabilities, found := s.capabilitiesReader.GetCapabilities(tenID) + if !found { + return conf, nil + } + + boundspb := capabilities.Cap(tenantcapabilities.TenantSpanConfigBounds).Get().Unwrap().(*tenantcapabilitiespb.SpanConfigBounds) + if boundspb == nil { + return conf, nil + } + bounds := spanconfigbounds.MakeBounds(boundspb) + + clamped := bounds.Clamp(&conf) + + if clamped { + log.VInfof(ctx, 3, "span config for tenant clamped to %v", conf) + } + return conf, nil } func (s *Store) getFallbackConfig() roachpb.SpanConfig { @@ -172,7 +227,7 @@ func (s *Store) Clone() *Store { s.mu.Lock() defer s.mu.Unlock() - clone := New(s.fallback, s.settings, s.knobs) + clone := New(s.fallback, s.settings, s.capabilitiesReader, s.knobs) clone.mu.spanConfigStore = s.mu.spanConfigStore.clone() clone.mu.systemSpanConfigStore = s.mu.systemSpanConfigStore.clone() return clone diff --git a/pkg/spanconfig/spanconfigstore/store_test.go b/pkg/spanconfig/spanconfigstore/store_test.go index 74fdd9596984..f6a9fab1a519 100644 --- a/pkg/spanconfig/spanconfigstore/store_test.go +++ b/pkg/spanconfig/spanconfigstore/store_test.go @@ -17,6 +17,7 @@ import ( "strings" "testing" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" @@ -104,6 +105,11 @@ func (s *spanConfigStore) TestingSplitKeys(tb testing.TB, start, end roachpb.RKe // A (refs = 2) // B (refs = 1) // +// declare-bounds +// set /Tenant/20:{GC.ttl_start=15, GC.ttl_end=30} +// delete /Tenant/10 +// ---- +// // Text of the form [a,b), {entire-keyspace}, {source=1,target=20}, and [a,b):C // correspond to targets {spans, system targets} and span config records; see // spanconfigtestutils.Parse{Target,Config,SpanConfigRecord} for more details. @@ -111,10 +117,14 @@ func TestDataDriven(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() + capabilitiesReader := newMockCapabilitiesReader() + settings := cluster.MakeTestingClusterSettings() + boundsEnabled.Override(ctx, &settings.SV, true) datadriven.Walk(t, datapathutils.TestDataPath(t), func(t *testing.T, path string) { store := New( spanconfigtestutils.ParseConfig(t, "FALLBACK"), - cluster.MakeTestingClusterSettings(), + settings, + capabilitiesReader, &spanconfig.TestingKnobs{ StoreIgnoreCoalesceAdjacentExceptions: true, StoreInternConfigsInDryRuns: true, @@ -147,7 +157,8 @@ func TestDataDriven(t *testing.T) { case "get": d.ScanArgs(t, "key", &keyStr) - config, err := store.GetSpanConfigForKey(ctx, roachpb.RKey(keyStr)) + key, _ := spanconfigtestutils.ParseKey(t, keyStr) + config, err := store.GetSpanConfigForKey(ctx, roachpb.RKey(key)) require.NoError(t, err) return fmt.Sprintf("conf=%s", spanconfigtestutils.PrintSpanConfig(config)) @@ -205,6 +216,9 @@ func TestDataDriven(t *testing.T) { } return b.String() + case "declare-bounds": + updates := spanconfigtestutils.ParseDeclareBoundsArguments(t, d.Input) + capabilitiesReader.apply(updates) default: t.Fatalf("unknown command: %s", d.Cmd) } @@ -214,6 +228,41 @@ func TestDataDriven(t *testing.T) { }) } +type mockCapabilitiesReader struct { + capabilities map[roachpb.TenantID]tenantcapabilities.TenantCapabilities +} + +func newMockCapabilitiesReader() *mockCapabilitiesReader { + m := mockCapabilitiesReader{ + capabilities: make(map[roachpb.TenantID]tenantcapabilities.TenantCapabilities), + } + return &m +} + +// GetCapabilities implements the tenantcapabilities.Reader interface. +func (m *mockCapabilitiesReader) GetCapabilities( + id roachpb.TenantID, +) (tenantcapabilities.TenantCapabilities, bool) { + caps, found := m.capabilities[id] + return caps, found +} + +// GetGlobalCapabilityState implements the tenantcapabilities.Reader interface. +func (m *mockCapabilitiesReader) GetGlobalCapabilityState() map[roachpb.TenantID]tenantcapabilities.TenantCapabilities { + panic("unimplemented") +} + +func (m *mockCapabilitiesReader) apply(updates []tenantcapabilities.Update) { + for _, update := range updates { + if update.Deleted { + delete(m.capabilities, update.TenantID) + continue + } + + m.capabilities[update.TenantID] = update.TenantCapabilities + } +} + // TestStoreClone verifies that a cloned store contains the same contents as the // original. func TestStoreClone(t *testing.T) { @@ -257,7 +306,12 @@ func TestStoreClone(t *testing.T) { ), } - original := New(roachpb.TestingDefaultSpanConfig(), cluster.MakeClusterSettings(), nil) + original := New( + roachpb.TestingDefaultSpanConfig(), + cluster.MakeClusterSettings(), + tenantcapabilities.NewEmptyReader(), + nil, + ) original.Apply(ctx, false, updates...) clone := original.Clone() @@ -293,6 +347,7 @@ func BenchmarkStoreComputeSplitKey(b *testing.B) { store := New( roachpb.SpanConfig{}, cluster.MakeClusterSettings(), + tenantcapabilities.NewEmptyReader(), &spanconfig.TestingKnobs{ StoreIgnoreCoalesceAdjacentExceptions: true, }, diff --git a/pkg/spanconfig/spanconfigstore/testdata/bounds/basic b/pkg/spanconfig/spanconfigstore/testdata/bounds/basic new file mode 100644 index 000000000000..109e3f169d56 --- /dev/null +++ b/pkg/spanconfig/spanconfigstore/testdata/bounds/basic @@ -0,0 +1,72 @@ +apply +set [/Tenant/10/a,/Tenant/10/b):GC.ttl=10 +set [/Tenant/10/f,/Tenant/10/h):GC.ttl=20 +set [/Tenant/10/h,/Tenant/10/j):GC.ttl=35 +---- +added [/Tenant/10/a,/Tenant/10/b):GC.ttl=10 +added [/Tenant/10/f,/Tenant/10/h):GC.ttl=20 +added [/Tenant/10/h,/Tenant/10/j):GC.ttl=35 + +apply +set [/Tenant/20/a,/Tenant/20/b):GC.ttl=55 +set [/Tenant/20/c,/Tenant/20/d):GC.ttl=25 +set [/Tenant/20/e,/Tenant/20/f):GC.ttl=3 +---- +added [/Tenant/20/a,/Tenant/20/b):GC.ttl=55 +added [/Tenant/20/c,/Tenant/20/d):GC.ttl=25 +added [/Tenant/20/e,/Tenant/20/f):GC.ttl=3 + +declare-bounds +set /Tenant/10:{GC.ttl_start=15, GC.ttl_end=30} +set /Tenant/20:{GC.ttl_start=5, GC.ttl_end=50} +---- + +# Ensure GC TTL is clamped correctly for tenant 10. +overlapping span=[/Tenant/10/a,/Tenant/10/j) +---- +[/Tenant/10/a,/Tenant/10/b):GC.ttl=15 +[/Tenant/10/f,/Tenant/10/h):GC.ttl=20 +[/Tenant/10/h,/Tenant/10/j):GC.ttl=30 + +get key=/Tenant/10/a +---- +conf=GC.ttl=15 + + +# Ditto for tenant 20. +overlapping span=[/Tenant/20/a,/Tenant/20/f) +---- +[/Tenant/20/a,/Tenant/20/b):GC.ttl=50 +[/Tenant/20/c,/Tenant/20/d):GC.ttl=25 +[/Tenant/20/e,/Tenant/20/f):GC.ttl=5 + +get key=/Tenant/20/e +---- +conf=GC.ttl=5 + +# Delete bounds for tenant 10. +declare-bounds +delete /Tenant/10 +---- + +# Ensure GC TTL values are no longer clamped for tenant 10. +overlapping span=[/Tenant/10/a,/Tenant/10/j) +---- +[/Tenant/10/a,/Tenant/10/b):GC.ttl=10 +[/Tenant/10/f,/Tenant/10/h):GC.ttl=20 +[/Tenant/10/h,/Tenant/10/j):GC.ttl=35 + +get key=/Tenant/10/a +---- +conf=GC.ttl=10 + +# However, this shouldn't affect tenant 20. +overlapping span=[/Tenant/20/a,/Tenant/20/f) +---- +[/Tenant/20/a,/Tenant/20/b):GC.ttl=50 +[/Tenant/20/c,/Tenant/20/d):GC.ttl=25 +[/Tenant/20/e,/Tenant/20/f):GC.ttl=5 + +get key=/Tenant/20/e +---- +conf=GC.ttl=5 diff --git a/pkg/spanconfig/spanconfigtestutils/BUILD.bazel b/pkg/spanconfig/spanconfigtestutils/BUILD.bazel index 2b9c5048a14f..850b8efb3154 100644 --- a/pkg/spanconfig/spanconfigtestutils/BUILD.bazel +++ b/pkg/spanconfig/spanconfigtestutils/BUILD.bazel @@ -14,6 +14,8 @@ go_library( "//pkg/keys", "//pkg/kv", "//pkg/kv/kvserver/protectedts/ptpb", + "//pkg/multitenant/tenantcapabilities", + "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb", "//pkg/roachpb", "//pkg/spanconfig", "//pkg/sql/catalog/descpb", diff --git a/pkg/spanconfig/spanconfigtestutils/utils.go b/pkg/spanconfig/spanconfigtestutils/utils.go index 771b66d5512b..c7e50c211b25 100644 --- a/pkg/spanconfig/spanconfigtestutils/utils.go +++ b/pkg/spanconfig/spanconfigtestutils/utils.go @@ -23,6 +23,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptpb" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/spanconfig" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" @@ -42,9 +44,13 @@ var systemTargetRe = regexp.MustCompile( `^{(entire-keyspace)|(source=(\d*),\s??((target=(\d*))|all-tenant-keyspace-targets-set))}$`, ) -// configRe matches either FALLBACK (for readability) or a single letter. It's a -// shorthand for declaring a unique tagged config. -var configRe = regexp.MustCompile(`^(FALLBACK)|(^\w)$`) +// configRe matches either FALLBACK (for readability), a single letter, or a +// specified GC TTL value. It's a shorthand for declaring a unique tagged config. +var configRe = regexp.MustCompile(`^(FALLBACK)|(^GC\.ttl=(\d*))|(^\w)$`) + +// boundsRe matches a string of the form "{GC.ttl_start=,GC.ttl_end=}". +// It translates to an upper/lower bound expressed through SpanConfigBounds. +var boundsRe = regexp.MustCompile(`{GC\.ttl_start=(\d*),\s??GC.ttl_end=(\d*)}`) // tenantRe matches a string of the form "/Tenant/". var tenantRe = regexp.MustCompile(`/Tenant/(\d*)`) @@ -239,8 +245,7 @@ func ParseTarget(t testing.TB, target string) spanconfig.Target { } // ParseConfig is helper function that constructs a roachpb.SpanConfig that's -// "tagged" with the given string (i.e. a constraint with the given string a -// required key). +// "tagged" with the given string. See configRe for specifics. func ParseConfig(t testing.TB, conf string) roachpb.SpanConfig { if !configRe.MatchString(conf) { t.Fatalf("expected %s to match config regex", conf) @@ -248,20 +253,31 @@ func ParseConfig(t testing.TB, conf string) roachpb.SpanConfig { matches := configRe.FindStringSubmatch(conf) var ts int64 + var gcTTL int if matches[1] == "FALLBACK" { ts = -1 + } else if matches[4] != "" { + ts = int64(matches[4][0]) } else { - ts = int64(matches[2][0]) + var err error + gcTTL, err = strconv.Atoi(matches[3]) + require.NoError(t, err) } - return roachpb.SpanConfig{ - GCPolicy: roachpb.GCPolicy{ - ProtectionPolicies: []roachpb.ProtectionPolicy{ - { - ProtectedTimestamp: hlc.Timestamp{ - WallTime: ts, - }, + + var protectionPolicies []roachpb.ProtectionPolicy + if ts != 0 { + protectionPolicies = []roachpb.ProtectionPolicy{ + { + ProtectedTimestamp: hlc.Timestamp{ + WallTime: ts, }, }, + } + } + return roachpb.SpanConfig{ + GCPolicy: roachpb.GCPolicy{ + ProtectionPolicies: protectionPolicies, + TTLSeconds: int32(gcTTL), }, } } @@ -518,6 +534,12 @@ func PrintTarget(t testing.TB, target spanconfig.Target) string { // ParseSpanConfig helper above. func PrintSpanConfig(config roachpb.SpanConfig) string { // See ParseConfig for what a "tagged" roachpb.SpanConfig translates to. + if config.GCPolicy.TTLSeconds != 0 && len(config.GCPolicy.ProtectionPolicies) != 0 { + panic("cannot have both TTL and protection policies set for tagged configs") // sanity check + } + if config.GCPolicy.TTLSeconds != 0 { + return fmt.Sprintf("GC.ttl=%d", config.GCPolicy.TTLSeconds) + } conf := make([]string, 0, len(config.GCPolicy.ProtectionPolicies)*2) for i, policy := range config.GCPolicy.ProtectionPolicies { if i > 0 { diff --git a/pkg/testutils/localtestcluster/BUILD.bazel b/pkg/testutils/localtestcluster/BUILD.bazel index d16746fc458d..2a975e7f191e 100644 --- a/pkg/testutils/localtestcluster/BUILD.bazel +++ b/pkg/testutils/localtestcluster/BUILD.bazel @@ -20,6 +20,7 @@ go_library( "//pkg/kv/kvserver/closedts/sidetransport", "//pkg/kv/kvserver/kvstorage", "//pkg/kv/kvserver/liveness", + "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/rpc", "//pkg/server/systemconfigwatcher", diff --git a/pkg/testutils/localtestcluster/local_test_cluster.go b/pkg/testutils/localtestcluster/local_test_cluster.go index 7d424d1d2b26..3868480fb75f 100644 --- a/pkg/testutils/localtestcluster/local_test_cluster.go +++ b/pkg/testutils/localtestcluster/local_test_cluster.go @@ -29,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvstorage" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/server/systemconfigwatcher" @@ -230,8 +231,9 @@ func (ltc *LocalTestCluster) Start(t testing.TB, baseCtx *base.Config, initFacto 1<<20, /* 1 MB */ cfg.DefaultSpanConfig, cfg.Settings, - nil, - nil, + tenantcapabilities.NewEmptyReader(), + nil, /* knobs */ + nil, /* registry */ ) cfg.SystemConfigProvider = systemconfigwatcher.New( keys.SystemSQLCodec, From 54e0952da93575e231e31ccb0d618937947fb570 Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Wed, 29 Mar 2023 21:02:06 +0000 Subject: [PATCH 4/4] spanconfig: use a narrower interface to get bounds from capabilities The spanconfig package doesn't care about any capabilities other than span config bounds which are stored as Capabilities. This patch introduces a new `spanconfigbounds.Reader` interface which is intended to be a narrow, adapter interface over `tenantcapabilities.Reader`. We then use this new interface in the `KVSubscriber` and `Store`. Epic: none Release note: None --- pkg/kv/kvserver/BUILD.bazel | 2 +- pkg/kv/kvserver/client_spanconfigs_test.go | 6 +- .../tenantcapabilities/capabilities.go | 18 ----- pkg/server/BUILD.bazel | 1 + pkg/server/server.go | 3 +- pkg/spanconfig/spanconfigbounds/BUILD.bazel | 2 + .../spanconfigbounds/bounds_reader.go | 66 +++++++++++++++++++ .../spanconfigkvsubscriber/BUILD.bazel | 4 +- .../spanconfigkvsubscriber/datadriven_test.go | 4 +- .../spanconfigkvsubscriber/kvsubscriber.go | 22 +++---- .../kvsubscriber_test.go | 4 +- .../spanconfigreconciler/BUILD.bazel | 2 +- .../spanconfigreconciler/reconciler.go | 4 +- pkg/spanconfig/spanconfigreporter/BUILD.bazel | 2 +- .../spanconfigreporter/datadriven_test.go | 4 +- pkg/spanconfig/spanconfigstore/BUILD.bazel | 4 +- pkg/spanconfig/spanconfigstore/store.go | 27 +++----- pkg/spanconfig/spanconfigstore/store_test.go | 43 +++++------- .../spanconfigtestutils/BUILD.bazel | 2 +- pkg/spanconfig/spanconfigtestutils/utils.go | 40 ++++++----- pkg/testutils/localtestcluster/BUILD.bazel | 2 +- .../localtestcluster/local_test_cluster.go | 4 +- 22 files changed, 153 insertions(+), 113 deletions(-) create mode 100644 pkg/spanconfig/spanconfigbounds/bounds_reader.go diff --git a/pkg/kv/kvserver/BUILD.bazel b/pkg/kv/kvserver/BUILD.bazel index 9e6f0ec2bab2..6b6fdb132ac7 100644 --- a/pkg/kv/kvserver/BUILD.bazel +++ b/pkg/kv/kvserver/BUILD.bazel @@ -389,7 +389,6 @@ go_test( "//pkg/kv/kvserver/tscache", "//pkg/kv/kvserver/txnwait", "//pkg/kv/kvserver/uncertainty", - "//pkg/multitenant/tenantcapabilities", "//pkg/multitenant/tenantcapabilities/tenantcapabilitiesauthorizer", "//pkg/roachpb", "//pkg/rpc", @@ -402,6 +401,7 @@ go_test( "//pkg/server/telemetry", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/spanconfig/spanconfigptsreader", "//pkg/spanconfig/spanconfigstore", "//pkg/sql", diff --git a/pkg/kv/kvserver/client_spanconfigs_test.go b/pkg/kv/kvserver/client_spanconfigs_test.go index 115704076ea0..970ff1ce86ed 100644 --- a/pkg/kv/kvserver/client_spanconfigs_test.go +++ b/pkg/kv/kvserver/client_spanconfigs_test.go @@ -18,10 +18,10 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigstore" "github.com/cockroachdb/cockroach/pkg/sql/isql" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" @@ -42,7 +42,7 @@ func TestSpanConfigUpdateAppliedToReplica(t *testing.T) { spanConfigStore := spanconfigstore.New( roachpb.TestingDefaultSpanConfig(), cluster.MakeTestingClusterSettings(), - tenantcapabilities.NewEmptyReader(), + spanconfigbounds.NewEmptyReader(), nil, ) var t0 = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) @@ -110,7 +110,7 @@ func TestFallbackSpanConfigOverride(t *testing.T) { st := cluster.MakeTestingClusterSettings() spanConfigStore := spanconfigstore.New( - roachpb.TestingDefaultSpanConfig(), st, tenantcapabilities.NewEmptyReader(), nil, + roachpb.TestingDefaultSpanConfig(), st, spanconfigbounds.NewEmptyReader(), nil, ) var t0 = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) mockSubscriber := newMockSpanConfigSubscriber(t0, spanConfigStore) diff --git a/pkg/multitenant/tenantcapabilities/capabilities.go b/pkg/multitenant/tenantcapabilities/capabilities.go index 528573b67e43..2a7c0056ddaa 100644 --- a/pkg/multitenant/tenantcapabilities/capabilities.go +++ b/pkg/multitenant/tenantcapabilities/capabilities.go @@ -249,21 +249,3 @@ func (c CapabilityID) CapabilityType() Type { panic(errors.AssertionFailedf("missing case: %q", c)) } } - -// NewEmptyReader returns a new Reader which corresponds to an empty global -// capability state. -func NewEmptyReader() Reader { - return emptyReader(true) -} - -type emptyReader bool - -// GetCapabilities implements the Reader interface. -func (emptyReader) GetCapabilities(roachpb.TenantID) (TenantCapabilities, bool) { - return nil, false -} - -// GetGlobalCapabilityState implements the Reader interface. -func (emptyReader) GetGlobalCapabilityState() map[roachpb.TenantID]TenantCapabilities { - return nil -} diff --git a/pkg/server/BUILD.bazel b/pkg/server/BUILD.bazel index 61145a5a385e..7d994e0397a9 100644 --- a/pkg/server/BUILD.bazel +++ b/pkg/server/BUILD.bazel @@ -178,6 +178,7 @@ go_library( "//pkg/settings", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/spanconfig/spanconfigjob", "//pkg/spanconfig/spanconfigkvaccessor", "//pkg/spanconfig/spanconfigkvsubscriber", diff --git a/pkg/server/server.go b/pkg/server/server.go index 48b8d359b63b..b2e95b8bf3de 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -71,6 +71,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/tenantsettingswatcher" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" _ "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigjob" // register jobs declared outside of pkg/sql "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvaccessor" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber" @@ -677,7 +678,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { 1<<20, /* 1 MB */ fallbackConf, cfg.Settings, - tenantCapabilitiesWatcher, + spanconfigbounds.NewReader(tenantCapabilitiesWatcher), spanConfigKnobs, registry, ) diff --git a/pkg/spanconfig/spanconfigbounds/BUILD.bazel b/pkg/spanconfig/spanconfigbounds/BUILD.bazel index a7891683fcb6..a46df79654ab 100644 --- a/pkg/spanconfig/spanconfigbounds/BUILD.bazel +++ b/pkg/spanconfig/spanconfigbounds/BUILD.bazel @@ -6,6 +6,7 @@ go_library( srcs = [ "bool_field.go", "bounds.go", + "bounds_reader.go", "constraints_field.go", "doc.go", "fields.go", @@ -21,6 +22,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/config", + "//pkg/multitenant/tenantcapabilities", "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb", "//pkg/roachpb", "//pkg/util/protoutil", diff --git a/pkg/spanconfig/spanconfigbounds/bounds_reader.go b/pkg/spanconfig/spanconfigbounds/bounds_reader.go new file mode 100644 index 000000000000..2d898ab8598e --- /dev/null +++ b/pkg/spanconfig/spanconfigbounds/bounds_reader.go @@ -0,0 +1,66 @@ +// Copyright 2023 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package spanconfigbounds + +import ( + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" + "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb" + "github.com/cockroachdb/cockroach/pkg/roachpb" +) + +// Reader maintains an in-memory view of the global SpanConfigBounds state. +// +// SpanConfigBounds are stored as tenant capabilities, the state of which is +// surfaced by the tenantcapabilities.Reader. BoundsReader serves as a narrow, +// adapter interface for the same. +type Reader interface { + // Bounds returns span config bounds set for a given tenant. If no bounds have + // been configured for the given tenant, found returns false. + Bounds(tenID roachpb.TenantID) (_ Bounds, found bool) +} + +type boundsReader struct { + capabilitiesReader tenantcapabilities.Reader +} + +// NewReader constructs and returns a new Reader. +func NewReader(reader tenantcapabilities.Reader) Reader { + return &boundsReader{ + capabilitiesReader: reader, + } +} + +// Bounds implements the BoundsReader interface. +func (r *boundsReader) Bounds(tenID roachpb.TenantID) (_ Bounds, found bool) { + capabilities, found := r.capabilitiesReader.GetCapabilities(tenID) + if !found { + return Bounds{}, false + } + + boundspb := capabilities.Cap(tenantcapabilities.TenantSpanConfigBounds).Get().Unwrap().(*tenantcapabilitiespb.SpanConfigBounds) + if boundspb == nil { + return Bounds{}, false + } + return MakeBounds(boundspb), true +} + +// NewEmptyReader returns a new Reader which corresponds to an empty span config +// bounds state. It's only intended for testing. +func NewEmptyReader() Reader { + return emptyReader(true) +} + +type emptyReader bool + +// Bounds implements the Reader interface. +func (emptyReader) Bounds(roachpb.TenantID) (Bounds, bool) { + return Bounds{}, false +} diff --git a/pkg/spanconfig/spanconfigkvsubscriber/BUILD.bazel b/pkg/spanconfig/spanconfigkvsubscriber/BUILD.bazel index cad8e24a3389..39f92766835d 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/BUILD.bazel +++ b/pkg/spanconfig/spanconfigkvsubscriber/BUILD.bazel @@ -16,11 +16,11 @@ go_library( "//pkg/kv/kvclient/rangefeed/rangefeedbuffer", "//pkg/kv/kvclient/rangefeed/rangefeedcache", "//pkg/kv/kvpb", - "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/settings", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/spanconfig/spanconfigstore", "//pkg/sql/catalog", "//pkg/sql/catalog/systemschema", @@ -58,13 +58,13 @@ go_test( "//pkg/kv/kvclient/rangefeed/rangefeedbuffer", "//pkg/kv/kvclient/rangefeed/rangefeedcache", "//pkg/kv/kvserver", - "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/security/securityassets", "//pkg/security/securitytest", "//pkg/server", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/spanconfig/spanconfigkvaccessor", "//pkg/spanconfig/spanconfigtestutils", "//pkg/sql/isql", diff --git a/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go b/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go index a895b51e7126..71843085eb46 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go +++ b/pkg/spanconfig/spanconfigkvsubscriber/datadriven_test.go @@ -21,9 +21,9 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedbuffer" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvaccessor" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigtestutils" @@ -148,7 +148,7 @@ func TestDataDriven(t *testing.T) { 10<<20, /* 10 MB */ spanconfigtestutils.ParseConfig(t, "FALLBACK"), tc.Server(0).ClusterSettings(), - tenantcapabilities.NewEmptyReader(), + spanconfigbounds.NewEmptyReader(), &spanconfig.TestingKnobs{ KVSubscriberRangeFeedKnobs: &rangefeedcache.TestingKnobs{ OnTimestampAdvance: func(ts hlc.Timestamp) { diff --git a/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go b/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go index 8e62ba0d6b77..fa5e1bf9532a 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go +++ b/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go @@ -18,11 +18,11 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedbuffer" "github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigstore" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -141,8 +141,8 @@ type KVSubscriber struct { clock *hlc.Clock metrics *Metrics - // capabilitiesReader provides a handle to the global tenant capability state. - capabilitiesReader tenantcapabilities.Reader + // boundsReader provides a handle to the global SpanConfigBounds state. + boundsReader spanconfigbounds.Reader } var _ spanconfig.KVSubscriber = &KVSubscriber{} @@ -190,7 +190,7 @@ func New( bufferMemLimit int64, fallback roachpb.SpanConfig, settings *cluster.Settings, - capabilitiesReader tenantcapabilities.Reader, + boundsReader spanconfigbounds.Reader, knobs *spanconfig.TestingKnobs, registry *metric.Registry, ) *KVSubscriber { @@ -205,13 +205,13 @@ func New( Key: spanConfigTableStart, EndKey: spanConfigTableStart.PrefixEnd(), } - spanConfigStore := spanconfigstore.New(fallback, settings, capabilitiesReader, knobs) + spanConfigStore := spanconfigstore.New(fallback, settings, boundsReader, knobs) s := &KVSubscriber{ - fallback: fallback, - knobs: knobs, - settings: settings, - clock: clock, - capabilitiesReader: capabilitiesReader, + fallback: fallback, + knobs: knobs, + settings: settings, + clock: clock, + boundsReader: boundsReader, } var rfCacheKnobs *rangefeedcache.TestingKnobs if knobs != nil { @@ -401,7 +401,7 @@ func (s *KVSubscriber) handleUpdate(ctx context.Context, u rangefeedcache.Update func (s *KVSubscriber) handleCompleteUpdate( ctx context.Context, ts hlc.Timestamp, events []rangefeedbuffer.Event, ) { - freshStore := spanconfigstore.New(s.fallback, s.settings, s.capabilitiesReader, s.knobs) + freshStore := spanconfigstore.New(s.fallback, s.settings, s.boundsReader, s.knobs) for _, ev := range events { freshStore.Apply(ctx, false /* dryrun */, ev.(*bufferEvent).Update) } diff --git a/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber_test.go b/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber_test.go index f7b0f0f07bf3..dedd8e8da693 100644 --- a/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber_test.go +++ b/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber_test.go @@ -16,10 +16,10 @@ import ( "testing" "github.com/cockroachdb/cockroach/pkg/keys" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -92,7 +92,7 @@ func TestGetProtectionTimestamps(t *testing.T) { 1<<20, /* 1 MB */ roachpb.SpanConfig{}, cluster.MakeTestingClusterSettings(), - tenantcapabilities.NewEmptyReader(), + spanconfigbounds.NewEmptyReader(), nil, nil, ) diff --git a/pkg/spanconfig/spanconfigreconciler/BUILD.bazel b/pkg/spanconfig/spanconfigreconciler/BUILD.bazel index 3102cdf4d913..f1590af59357 100644 --- a/pkg/spanconfig/spanconfigreconciler/BUILD.bazel +++ b/pkg/spanconfig/spanconfigreconciler/BUILD.bazel @@ -9,10 +9,10 @@ go_library( deps = [ "//pkg/keys", "//pkg/kv", - "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/spanconfig/spanconfigsqltranslator", "//pkg/spanconfig/spanconfigstore", "//pkg/sql", diff --git a/pkg/spanconfig/spanconfigreconciler/reconciler.go b/pkg/spanconfig/spanconfigreconciler/reconciler.go index e1c2b8e735b3..fd2975ca8371 100644 --- a/pkg/spanconfig/spanconfigreconciler/reconciler.go +++ b/pkg/spanconfig/spanconfigreconciler/reconciler.go @@ -16,10 +16,10 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqltranslator" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigstore" "github.com/cockroachdb/cockroach/pkg/sql" @@ -387,7 +387,7 @@ func (f *fullReconciler) fetchExistingSpanConfigs( } // The reconciler doesn't do any bounds checks or clamping, so it shouldn't // need access to tenant capabilities (and by extension span config bounds). - store := spanconfigstore.New(roachpb.SpanConfig{}, f.settings, tenantcapabilities.NewEmptyReader(), f.knobs) + store := spanconfigstore.New(roachpb.SpanConfig{}, f.settings, spanconfigbounds.NewEmptyReader(), f.knobs) { // Fully populate the store with KVAccessor contents. records, err := f.kvAccessor.GetSpanConfigRecords(ctx, targets) diff --git a/pkg/spanconfig/spanconfigreporter/BUILD.bazel b/pkg/spanconfig/spanconfigreporter/BUILD.bazel index edf12d3efd7a..1f75c9d87cc3 100644 --- a/pkg/spanconfig/spanconfigreporter/BUILD.bazel +++ b/pkg/spanconfig/spanconfigreporter/BUILD.bazel @@ -35,13 +35,13 @@ go_test( "//pkg/keys", "//pkg/kv/kvserver/constraint", "//pkg/kv/kvserver/liveness/livenesspb", - "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/security/securityassets", "//pkg/security/securitytest", "//pkg/server", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/spanconfig/spanconfigstore", "//pkg/spanconfig/spanconfigtestutils", "//pkg/testutils/datapathutils", diff --git a/pkg/spanconfig/spanconfigreporter/datadriven_test.go b/pkg/spanconfig/spanconfigreporter/datadriven_test.go index 5ca84d7001e6..af6ebaf9bb28 100644 --- a/pkg/spanconfig/spanconfigreporter/datadriven_test.go +++ b/pkg/spanconfig/spanconfigreporter/datadriven_test.go @@ -20,10 +20,10 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/constraint" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" clustersettings "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreporter" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigstore" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigtestutils" @@ -235,7 +235,7 @@ func newMockCluster( ranges: make(map[roachpb.RangeID]roachpb.RangeDescriptor), liveness: make(map[roachpb.NodeID]bool), store: spanconfigstore.New( - roachpb.TestingDefaultSpanConfig(), st, tenantcapabilities.NewEmptyReader(), scKnobs, + roachpb.TestingDefaultSpanConfig(), st, spanconfigbounds.NewEmptyReader(), scKnobs, ), } } diff --git a/pkg/spanconfig/spanconfigstore/BUILD.bazel b/pkg/spanconfig/spanconfigstore/BUILD.bazel index 38b774a78d94..238d99947d9c 100644 --- a/pkg/spanconfig/spanconfigstore/BUILD.bazel +++ b/pkg/spanconfig/spanconfigstore/BUILD.bazel @@ -15,8 +15,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/keys", - "//pkg/multitenant/tenantcapabilities", - "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb", "//pkg/roachpb", "//pkg/settings", "//pkg/settings/cluster", @@ -44,10 +42,10 @@ go_test( embed = [":spanconfigstore"], deps = [ "//pkg/keys", - "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/spanconfig/spanconfigtestutils", "//pkg/testutils/datapathutils", "//pkg/util/hlc", diff --git a/pkg/spanconfig/spanconfigstore/store.go b/pkg/spanconfig/spanconfigstore/store.go index 498f79aff207..9321728ca8ee 100644 --- a/pkg/spanconfig/spanconfigstore/store.go +++ b/pkg/spanconfig/spanconfigstore/store.go @@ -14,8 +14,6 @@ import ( "context" "github.com/cockroachdb/cockroach/pkg/keys" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -88,8 +86,8 @@ type Store struct { knobs *spanconfig.TestingKnobs - // capabilitiesReader provides a handle to the global tenant capability state. - capabilitiesReader tenantcapabilities.Reader + // boundsReader provides a handle to the global SpanConfigBounds state. + boundsReader spanconfigbounds.Reader } var _ spanconfig.Store = &Store{} @@ -98,17 +96,17 @@ var _ spanconfig.Store = &Store{} func New( fallback roachpb.SpanConfig, settings *cluster.Settings, - capabilitiesReader tenantcapabilities.Reader, + boundsReader spanconfigbounds.Reader, knobs *spanconfig.TestingKnobs, ) *Store { if knobs == nil { knobs = &spanconfig.TestingKnobs{} } s := &Store{ - settings: settings, - fallback: fallback, - capabilitiesReader: capabilitiesReader, - knobs: knobs, + settings: settings, + fallback: fallback, + boundsReader: boundsReader, + knobs: knobs, } s.mu.spanConfigStore = newSpanConfigStore(settings, s.knobs) s.mu.systemSpanConfigStore = newSystemSpanConfigStore() @@ -170,16 +168,11 @@ func (s *Store) getSpanConfigForKeyRLocked( // SpanConfig bounds do not apply to the system tenant. return conf, nil } - capabilities, found := s.capabilitiesReader.GetCapabilities(tenID) - if !found { - return conf, nil - } - boundspb := capabilities.Cap(tenantcapabilities.TenantSpanConfigBounds).Get().Unwrap().(*tenantcapabilitiespb.SpanConfigBounds) - if boundspb == nil { + bounds, found := s.boundsReader.Bounds(tenID) + if !found { return conf, nil } - bounds := spanconfigbounds.MakeBounds(boundspb) clamped := bounds.Clamp(&conf) @@ -227,7 +220,7 @@ func (s *Store) Clone() *Store { s.mu.Lock() defer s.mu.Unlock() - clone := New(s.fallback, s.settings, s.capabilitiesReader, s.knobs) + clone := New(s.fallback, s.settings, s.boundsReader, s.knobs) clone.mu.spanConfigStore = s.mu.spanConfigStore.clone() clone.mu.systemSpanConfigStore = s.mu.systemSpanConfigStore.clone() return clone diff --git a/pkg/spanconfig/spanconfigstore/store_test.go b/pkg/spanconfig/spanconfigstore/store_test.go index f6a9fab1a519..e598246d732a 100644 --- a/pkg/spanconfig/spanconfigstore/store_test.go +++ b/pkg/spanconfig/spanconfigstore/store_test.go @@ -17,10 +17,10 @@ import ( "strings" "testing" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigtestutils" "github.com/cockroachdb/cockroach/pkg/testutils/datapathutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -117,14 +117,14 @@ func TestDataDriven(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() - capabilitiesReader := newMockCapabilitiesReader() + boundsReader := newMockBoundsReader() settings := cluster.MakeTestingClusterSettings() boundsEnabled.Override(ctx, &settings.SV, true) datadriven.Walk(t, datapathutils.TestDataPath(t), func(t *testing.T, path string) { store := New( spanconfigtestutils.ParseConfig(t, "FALLBACK"), settings, - capabilitiesReader, + boundsReader, &spanconfig.TestingKnobs{ StoreIgnoreCoalesceAdjacentExceptions: true, StoreInternConfigsInDryRuns: true, @@ -218,7 +218,7 @@ func TestDataDriven(t *testing.T) { case "declare-bounds": updates := spanconfigtestutils.ParseDeclareBoundsArguments(t, d.Input) - capabilitiesReader.apply(updates) + boundsReader.apply(updates) default: t.Fatalf("unknown command: %s", d.Cmd) } @@ -228,38 +228,31 @@ func TestDataDriven(t *testing.T) { }) } -type mockCapabilitiesReader struct { - capabilities map[roachpb.TenantID]tenantcapabilities.TenantCapabilities +type mockBoundsReader struct { + bounds map[roachpb.TenantID]spanconfigbounds.Bounds } -func newMockCapabilitiesReader() *mockCapabilitiesReader { - m := mockCapabilitiesReader{ - capabilities: make(map[roachpb.TenantID]tenantcapabilities.TenantCapabilities), +func newMockBoundsReader() *mockBoundsReader { + m := mockBoundsReader{ + bounds: make(map[roachpb.TenantID]spanconfigbounds.Bounds), } return &m } -// GetCapabilities implements the tenantcapabilities.Reader interface. -func (m *mockCapabilitiesReader) GetCapabilities( - id roachpb.TenantID, -) (tenantcapabilities.TenantCapabilities, bool) { - caps, found := m.capabilities[id] - return caps, found +// Bounds implements the spanconfigbounds.Reader interface. +func (m *mockBoundsReader) Bounds(id roachpb.TenantID) (spanconfigbounds.Bounds, bool) { + bounds, found := m.bounds[id] + return bounds, found } -// GetGlobalCapabilityState implements the tenantcapabilities.Reader interface. -func (m *mockCapabilitiesReader) GetGlobalCapabilityState() map[roachpb.TenantID]tenantcapabilities.TenantCapabilities { - panic("unimplemented") -} - -func (m *mockCapabilitiesReader) apply(updates []tenantcapabilities.Update) { +func (m *mockBoundsReader) apply(updates []spanconfigtestutils.BoundsUpdate) { for _, update := range updates { if update.Deleted { - delete(m.capabilities, update.TenantID) + delete(m.bounds, update.TenantID) continue } - m.capabilities[update.TenantID] = update.TenantCapabilities + m.bounds[update.TenantID] = update.Bounds } } @@ -309,7 +302,7 @@ func TestStoreClone(t *testing.T) { original := New( roachpb.TestingDefaultSpanConfig(), cluster.MakeClusterSettings(), - tenantcapabilities.NewEmptyReader(), + spanconfigbounds.NewEmptyReader(), nil, ) original.Apply(ctx, false, updates...) @@ -347,7 +340,7 @@ func BenchmarkStoreComputeSplitKey(b *testing.B) { store := New( roachpb.SpanConfig{}, cluster.MakeClusterSettings(), - tenantcapabilities.NewEmptyReader(), + spanconfigbounds.NewEmptyReader(), &spanconfig.TestingKnobs{ StoreIgnoreCoalesceAdjacentExceptions: true, }, diff --git a/pkg/spanconfig/spanconfigtestutils/BUILD.bazel b/pkg/spanconfig/spanconfigtestutils/BUILD.bazel index 850b8efb3154..773614f5cbc1 100644 --- a/pkg/spanconfig/spanconfigtestutils/BUILD.bazel +++ b/pkg/spanconfig/spanconfigtestutils/BUILD.bazel @@ -14,10 +14,10 @@ go_library( "//pkg/keys", "//pkg/kv", "//pkg/kv/kvserver/protectedts/ptpb", - "//pkg/multitenant/tenantcapabilities", "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb", "//pkg/roachpb", "//pkg/spanconfig", + "//pkg/spanconfig/spanconfigbounds", "//pkg/sql/catalog/descpb", "//pkg/util/hlc", "//pkg/util/syncutil", diff --git a/pkg/spanconfig/spanconfigtestutils/utils.go b/pkg/spanconfig/spanconfigtestutils/utils.go index c7e50c211b25..c2e8ddd86955 100644 --- a/pkg/spanconfig/spanconfigtestutils/utils.go +++ b/pkg/spanconfig/spanconfigtestutils/utils.go @@ -23,10 +23,10 @@ import ( "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptpb" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities/tenantcapabilitiespb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/datadriven" @@ -282,14 +282,20 @@ func ParseConfig(t testing.TB, conf string) roachpb.SpanConfig { } } +// BoundsUpdate .. +type BoundsUpdate struct { + TenantID roachpb.TenantID + Bounds spanconfigbounds.Bounds + Deleted bool +} + // ParseDeclareBoundsArguments parses datadriven test input to a list of // tenantcapabilities.Updates that can be applied to a // tenantcapabilities.Reader. The input is of the following form: // // delete ten-10 // set ten-10:{GC.ttl_start=5, GC.ttl_end=30} -func ParseDeclareBoundsArguments(t *testing.T, input string) []tenantcapabilities.Update { - var updates []tenantcapabilities.Update +func ParseDeclareBoundsArguments(t *testing.T, input string) (updates []BoundsUpdate) { for _, line := range strings.Split(input, "\n") { line = strings.TrimSpace(line) if line == "" { @@ -305,23 +311,21 @@ func ParseDeclareBoundsArguments(t *testing.T, input string) []tenantcapabilitie require.Len(t, parts, 2) tenantID := parseTenant(t, parts[0]) bounds := parseBounds(t, parts[1]) - updates = append(updates, tenantcapabilities.Update{ - Entry: tenantcapabilities.Entry{ - TenantID: tenantID, - TenantCapabilities: &tenantcapabilitiespb.TenantCapabilities{ - SpanConfigBounds: bounds, - }, - }, + updates = append(updates, BoundsUpdate{ + TenantID: tenantID, + Bounds: bounds, }) + case strings.HasPrefix(line, deletePrefix): line = strings.TrimPrefix(line, line[:len(deletePrefix)]) tenantID := parseTenant(t, line) - updates = append(updates, tenantcapabilities.Update{ - Entry: tenantcapabilities.Entry{ + updates = append(updates, + BoundsUpdate{ TenantID: tenantID, + Bounds: spanconfigbounds.Bounds{}, + Deleted: true, }, - Deleted: true, - }) + ) default: t.Fatalf("malformed line %q, expected to find prefix %q or %q", line, setPrefix, deletePrefix) @@ -331,8 +335,8 @@ func ParseDeclareBoundsArguments(t *testing.T, input string) []tenantcapabilitie } // parseBounds parses a string that looks like {GC.ttl_start=5, GC.ttl_end=40} -// into a SpanConfigBoudns struct. -func parseBounds(t *testing.T, input string) *tenantcapabilitiespb.SpanConfigBounds { +// into a spanconfigbounds.Bounds struct. +func parseBounds(t *testing.T, input string) spanconfigbounds.Bounds { require.True(t, boundsRe.MatchString(input)) matches := boundsRe.FindStringSubmatch(input) @@ -341,12 +345,12 @@ func parseBounds(t *testing.T, input string) *tenantcapabilitiespb.SpanConfigBou gcTTLEnd, err := strconv.Atoi(matches[2]) require.NoError(t, err) - return &tenantcapabilitiespb.SpanConfigBounds{ + return spanconfigbounds.MakeBounds(&tenantcapabilitiespb.SpanConfigBounds{ GCTTLSeconds: &tenantcapabilitiespb.SpanConfigBounds_Int32Range{ Start: int32(gcTTLStart), End: int32(gcTTLEnd), }, - } + }) } // ParseSpanConfigRecord is helper function that constructs a diff --git a/pkg/testutils/localtestcluster/BUILD.bazel b/pkg/testutils/localtestcluster/BUILD.bazel index 2a975e7f191e..037d6d1467d2 100644 --- a/pkg/testutils/localtestcluster/BUILD.bazel +++ b/pkg/testutils/localtestcluster/BUILD.bazel @@ -20,11 +20,11 @@ go_library( "//pkg/kv/kvserver/closedts/sidetransport", "//pkg/kv/kvserver/kvstorage", "//pkg/kv/kvserver/liveness", - "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", "//pkg/rpc", "//pkg/server/systemconfigwatcher", "//pkg/settings/cluster", + "//pkg/spanconfig/spanconfigbounds", "//pkg/spanconfig/spanconfigkvsubscriber", "//pkg/sql/catalog/bootstrap", "//pkg/storage", diff --git a/pkg/testutils/localtestcluster/local_test_cluster.go b/pkg/testutils/localtestcluster/local_test_cluster.go index 3868480fb75f..10a30c06b17a 100644 --- a/pkg/testutils/localtestcluster/local_test_cluster.go +++ b/pkg/testutils/localtestcluster/local_test_cluster.go @@ -29,11 +29,11 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvstorage" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness" - "github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/server/systemconfigwatcher" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigbounds" "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber" "github.com/cockroachdb/cockroach/pkg/sql/catalog/bootstrap" "github.com/cockroachdb/cockroach/pkg/storage" @@ -231,7 +231,7 @@ func (ltc *LocalTestCluster) Start(t testing.TB, baseCtx *base.Config, initFacto 1<<20, /* 1 MB */ cfg.DefaultSpanConfig, cfg.Settings, - tenantcapabilities.NewEmptyReader(), + spanconfigbounds.NewEmptyReader(), nil, /* knobs */ nil, /* registry */ )