From 98ab46127ea09c11c9c50d5cb7319cae421d24dd Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Fri, 4 Nov 2022 19:19:36 -0400 Subject: [PATCH 1/2] rangedesciter: support scoped iteration Informs #87503; adds the ability to only paginate through range descriptors in the system that overlap with some given span. We're going to use it in future commits as part of multi-tenant replication reports (#89987) where we'll need to iterate over the set of range descriptors that pertain only to some span of interest (for a given index, partition, tenant, etc.) Release note: None --- pkg/kv/kvclient/kvcoord/batch_test.go | 2 +- pkg/kv/kvclient/kvcoord/dist_sender.go | 4 +- .../kvclient/kvcoord/dist_sender_rangefeed.go | 2 +- pkg/kv/kvclient/kvstreamer/streamer.go | 2 +- pkg/kv/kvclient/rangecache/range_cache.go | 4 +- pkg/kv/kvclient/rangefeed/db_adapter.go | 2 +- pkg/roachpb/data.go | 18 +-- pkg/roachpb/data_test.go | 4 +- pkg/upgrade/upgradecluster/BUILD.bazel | 1 + pkg/upgrade/upgradecluster/cluster.go | 3 +- pkg/util/rangedesciter/BUILD.bazel | 5 + pkg/util/rangedesciter/rangedesciter.go | 95 ++++++++--- pkg/util/rangedesciter/rangedesciter_test.go | 137 ++++++++++++++-- .../rangedesciter/testdata/scoped_iteration | 149 ++++++++++++++++++ .../testdata/scoped_iteration_with_page_size | 80 ++++++++++ .../testdata/scoped_iteration_with_splits | 78 +++++++++ 16 files changed, 530 insertions(+), 56 deletions(-) create mode 100644 pkg/util/rangedesciter/testdata/scoped_iteration create mode 100644 pkg/util/rangedesciter/testdata/scoped_iteration_with_page_size create mode 100644 pkg/util/rangedesciter/testdata/scoped_iteration_with_splits diff --git a/pkg/kv/kvclient/kvcoord/batch_test.go b/pkg/kv/kvclient/kvcoord/batch_test.go index 76dce95f7891..1c199d0cfa27 100644 --- a/pkg/kv/kvclient/kvcoord/batch_test.go +++ b/pkg/kv/kvclient/kvcoord/batch_test.go @@ -432,7 +432,7 @@ func TestTruncate(t *testing.T) { desc.EndKey = roachpb.RKey(test.to) } rs := roachpb.RSpan{Key: roachpb.RKey(test.from), EndKey: roachpb.RKey(test.to)} - rs, err := rs.Intersect(desc) + rs, err := rs.Intersect(desc.RSpan()) if err != nil { t.Errorf("%d: intersection failure: %v", i, err) continue diff --git a/pkg/kv/kvclient/kvcoord/dist_sender.go b/pkg/kv/kvclient/kvcoord/dist_sender.go index 8587a21bec57..1e48755b30c2 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender.go @@ -1382,7 +1382,7 @@ func (ds *DistSender) divideAndSendBatchToRanges( responseChs = append(responseChs, responseCh) // Truncate the request to range descriptor. - curRangeRS, err := rs.Intersect(ri.Token().Desc()) + curRangeRS, err := rs.Intersect(ri.Token().Desc().RSpan()) if err != nil { responseCh <- response{pErr: roachpb.NewError(err)} return @@ -1632,7 +1632,7 @@ func (ds *DistSender) sendPartialBatch( // batch, so that we know that the response to it matches the positions // into our batch (using the full batch here would give a potentially // larger response slice with unknown mapping to our truncated reply). - intersection, err := rs.Intersect(routingTok.Desc()) + intersection, err := rs.Intersect(routingTok.Desc().RSpan()) if err != nil { return response{pErr: roachpb.NewError(err)} } diff --git a/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go b/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go index 7bd8888e9c2a..c7032d18fc46 100644 --- a/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go +++ b/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go @@ -345,7 +345,7 @@ func (ds *DistSender) divideAndSendRangeFeedToRanges( ri := MakeRangeIterator(ds) for ri.Seek(ctx, nextRS.Key, Ascending); ri.Valid(); ri.Next(ctx) { desc := ri.Desc() - partialRS, err := nextRS.Intersect(desc) + partialRS, err := nextRS.Intersect(desc.RSpan()) if err != nil { return err } diff --git a/pkg/kv/kvclient/kvstreamer/streamer.go b/pkg/kv/kvclient/kvstreamer/streamer.go index cf014e0caa8c..54ffd8abbd0e 100644 --- a/pkg/kv/kvclient/kvstreamer/streamer.go +++ b/pkg/kv/kvclient/kvstreamer/streamer.go @@ -551,7 +551,7 @@ func (s *Streamer) Enqueue(ctx context.Context, reqs []roachpb.RequestUnion) (re rs.Key = roachpb.RKeyMax } else { // Truncate the request span to the current range. - singleRangeSpan, err := rs.Intersect(ri.Token().Desc()) + singleRangeSpan, err := rs.Intersect(ri.Token().Desc().RSpan()) if err != nil { return err } diff --git a/pkg/kv/kvclient/rangecache/range_cache.go b/pkg/kv/kvclient/rangecache/range_cache.go index fe566d5ae521..ff04d3a10058 100644 --- a/pkg/kv/kvclient/rangecache/range_cache.go +++ b/pkg/kv/kvclient/rangecache/range_cache.go @@ -1424,7 +1424,7 @@ func (e *CacheEntry) LeaseSpeculative() bool { // "speculative" (sequence=0). func (e *CacheEntry) overrides(o *CacheEntry) bool { if util.RaceEnabled { - if _, err := e.Desc().RSpan().Intersect(o.Desc()); err != nil { + if _, err := e.Desc().RSpan().Intersect(o.Desc().RSpan()); err != nil { panic(fmt.Sprintf("descriptors don't intersect: %s vs %s", e.Desc(), o.Desc())) } } @@ -1464,7 +1464,7 @@ func (e *CacheEntry) overrides(o *CacheEntry) bool { // older; this matches the semantics of b.overrides(a). func compareEntryDescs(a, b *CacheEntry) int { if util.RaceEnabled { - if _, err := a.Desc().RSpan().Intersect(b.Desc()); err != nil { + if _, err := a.Desc().RSpan().Intersect(b.Desc().RSpan()); err != nil { panic(fmt.Sprintf("descriptors don't intersect: %s vs %s", a.Desc(), b.Desc())) } } diff --git a/pkg/kv/kvclient/rangefeed/db_adapter.go b/pkg/kv/kvclient/rangefeed/db_adapter.go index 7bcb92110fc9..67e35d410b15 100644 --- a/pkg/kv/kvclient/rangefeed/db_adapter.go +++ b/pkg/kv/kvclient/rangefeed/db_adapter.go @@ -253,7 +253,7 @@ func (dbc *dbAdapter) divideAndSendScanRequests( for ri.Seek(ctx, nextRS.Key, kvcoord.Ascending); ri.Valid(); ri.Next(ctx) { desc := ri.Desc() - partialRS, err := nextRS.Intersect(desc) + partialRS, err := nextRS.Intersect(desc.RSpan()) if err != nil { return err } diff --git a/pkg/roachpb/data.go b/pkg/roachpb/data.go index 7fc9122aa3d4..d7ab235b6caf 100644 --- a/pkg/roachpb/data.go +++ b/pkg/roachpb/data.go @@ -2395,20 +2395,20 @@ func (rs RSpan) String() string { } // Intersect returns the intersection of the current span and the -// descriptor's range. Returns an error if the span and the -// descriptor's range do not overlap. -func (rs RSpan) Intersect(desc *RangeDescriptor) (RSpan, error) { - if !rs.Key.Less(desc.EndKey) || !desc.StartKey.Less(rs.EndKey) { - return rs, errors.Errorf("span and descriptor's range do not overlap: %s vs %s", rs, desc) +// given range. Returns an error if the span and the range do not +// overlap. +func (rs RSpan) Intersect(rspan RSpan) (RSpan, error) { + if !rs.Key.Less(rspan.EndKey) || !rspan.Key.Less(rs.EndKey) { + return rs, errors.Errorf("spans do not overlap: %s vs %s", rs, rspan) } key := rs.Key - if key.Less(desc.StartKey) { - key = desc.StartKey + if key.Less(rspan.Key) { + key = rspan.Key } endKey := rs.EndKey - if !desc.ContainsKeyRange(desc.StartKey, endKey) { - endKey = desc.EndKey + if !rspan.ContainsKeyRange(rspan.Key, endKey) { + endKey = rspan.EndKey } return RSpan{key, endKey}, nil } diff --git a/pkg/roachpb/data_test.go b/pkg/roachpb/data_test.go index 0c553623bf88..ac8e1616b3e6 100644 --- a/pkg/roachpb/data_test.go +++ b/pkg/roachpb/data_test.go @@ -1482,7 +1482,7 @@ func TestRSpanIntersect(t *testing.T) { desc.StartKey = test.startKey desc.EndKey = test.endKey - actual, err := rs.Intersect(&desc) + actual, err := rs.Intersect(desc.RSpan()) if err != nil { t.Error(err) continue @@ -1504,7 +1504,7 @@ func TestRSpanIntersect(t *testing.T) { desc := RangeDescriptor{} desc.StartKey = test.startKey desc.EndKey = test.endKey - if _, err := rs.Intersect(&desc); err == nil { + if _, err := rs.Intersect(desc.RSpan()); err == nil { t.Errorf("%d: unexpected success", i) } } diff --git a/pkg/upgrade/upgradecluster/BUILD.bazel b/pkg/upgrade/upgradecluster/BUILD.bazel index ed9ddb606aee..9af78e897740 100644 --- a/pkg/upgrade/upgradecluster/BUILD.bazel +++ b/pkg/upgrade/upgradecluster/BUILD.bazel @@ -11,6 +11,7 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/upgrade/upgradecluster", visibility = ["//visibility:public"], deps = [ + "//pkg/keys", "//pkg/kv", "//pkg/kv/kvserver/liveness/livenesspb", "//pkg/roachpb", diff --git a/pkg/upgrade/upgradecluster/cluster.go b/pkg/upgrade/upgradecluster/cluster.go index d1630a92b197..07ac49742264 100644 --- a/pkg/upgrade/upgradecluster/cluster.go +++ b/pkg/upgrade/upgradecluster/cluster.go @@ -14,6 +14,7 @@ package upgradecluster import ( "context" + "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -145,5 +146,5 @@ func (c *Cluster) ForEveryNode( func (c *Cluster) IterateRangeDescriptors( ctx context.Context, blockSize int, init func(), fn func(...roachpb.RangeDescriptor) error, ) error { - return c.c.RangeDescIterator.Iterate(ctx, blockSize, init, fn) + return c.c.RangeDescIterator.Iterate(ctx, blockSize, init, keys.EverythingSpan, fn) } diff --git a/pkg/util/rangedesciter/BUILD.bazel b/pkg/util/rangedesciter/BUILD.bazel index aed75899a35f..bc5ec2c8c58c 100644 --- a/pkg/util/rangedesciter/BUILD.bazel +++ b/pkg/util/rangedesciter/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//pkg/keys", "//pkg/kv", "//pkg/roachpb", + "//pkg/util/iterutil", "@com_github_cockroachdb_errors//:errors", ], ) @@ -21,6 +22,7 @@ go_test( "rangedesciter_test.go", ], args = ["-test.timeout=295s"], + data = glob(["testdata/**"]), deps = [ ":rangedesciter", "//pkg/keys", @@ -30,9 +32,12 @@ go_test( "//pkg/security/securitytest", "//pkg/server", "//pkg/sql/tests", + "//pkg/testutils", "//pkg/testutils/serverutils", "//pkg/testutils/testcluster", "//pkg/util/leaktest", + "@com_github_cockroachdb_datadriven//:datadriven", + "@com_github_stretchr_testify//require", ], ) diff --git a/pkg/util/rangedesciter/rangedesciter.go b/pkg/util/rangedesciter/rangedesciter.go index 315516f04d72..474788c91793 100644 --- a/pkg/util/rangedesciter/rangedesciter.go +++ b/pkg/util/rangedesciter/rangedesciter.go @@ -16,15 +16,17 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/util/iterutil" "github.com/cockroachdb/errors" ) -// Iterator paginates through every range descriptor in the system. +// Iterator paginates through range descriptors in the system. type Iterator interface { - // Iterate paginates through range descriptors in the system using the given - // page size. It's important to note that the closure is being executed in - // the context of a distributed transaction that may be automatically - // retried. So something like the following is an anti-pattern: + // Iterate paginates through range descriptors in the system that overlap + // with the given span. When doing so it uses the given page size. It's + // important to note that the closure is being executed in the context of a + // distributed transaction that may be automatically retried. So something + // like the following is an anti-pattern: // // processed := 0 // _ = rdi.Iterate(..., @@ -45,8 +47,15 @@ type Iterator interface { // log.Infof(ctx, "processed %d ranges", processed) // }, // ) + // + // + // When the query span is something other than keys.EverythingSpan, the page + // size is also approximately haw many extra keys/range descriptors we may + // be reading. Callers are expected to pick a page size accordingly + // (page sizes that are much larger than expected # of descriptors would + // lead to wasted work). Iterate( - ctx context.Context, pageSize int, init func(), + ctx context.Context, pageSize int, init func(), span roachpb.Span, fn func(descriptors ...roachpb.RangeDescriptor) error, ) error } @@ -74,31 +83,58 @@ func (i *iteratorImpl) Iterate( ctx context.Context, pageSize int, init func(), + span roachpb.Span, fn func(descriptors ...roachpb.RangeDescriptor) error, ) error { + rspan := roachpb.RSpan{ + Key: keys.MustAddr(span.Key), + EndKey: keys.MustAddr(span.EndKey), + } + return i.db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error { // Inform the caller that we're starting a fresh attempt to page in // range descriptors. init() - // Iterate through meta{1,2} to pull out all the range descriptors. + // Bound the start key of the meta{1,2} scan as much as possible. If the + // start key < keys.Meta1KeyMax (we're also interested in the meta1 + // range descriptor), start our scan at keys.MetaMin. If not, start it + // at the relevant range meta key -- in meta1 if the start key sits + // within meta2, in meta2 if the start key is an ordinary key. + // + // So what exactly is the "relevant range meta key"? Since keys in meta + // ranges are encoded using the end keys of range descriptors, we're + // looking for the lowest existing range meta key that's strictly + // greater than RangeMetaKey(start key). + rangeMetaKeyForStart := keys.RangeMetaKey(rspan.Key) + metaScanBoundsForStart, err := keys.MetaScanBounds(rangeMetaKeyForStart) + if err != nil { + return err + } + metaScanStartKey := metaScanBoundsForStart.Key.AsRawKey() + + // Iterate through meta{1,2} to pull out relevant range descriptors. + // We'll keep scanning until we've found a range descriptor outside the + // scan of interest. var lastRangeIDInMeta1 roachpb.RangeID - return txn.Iterate(ctx, keys.MetaMin, keys.MetaMax, pageSize, + return iterutil.Map(txn.Iterate(ctx, metaScanStartKey, keys.MetaMax, pageSize, func(rows []kv.KeyValue) error { descriptors := make([]roachpb.RangeDescriptor, 0, len(rows)) + stopMetaIteration := false + var desc roachpb.RangeDescriptor for _, row := range rows { - err := row.ValueProto(&desc) - if err != nil { + if err := row.ValueProto(&desc); err != nil { return errors.Wrapf(err, "unable to unmarshal range descriptor from %s", row.Key) } - // In small enough clusters it's possible for the same range - // descriptor to be stored in both meta1 and meta2. This - // happens when some range spans both the meta and the user - // keyspace. Consider when r1 is [/Min, - // /System/NodeLiveness); we'll store the range descriptor - // in both /Meta2/ and in /Meta1/KeyMax[1]. + // In small enough clusters, it's possible for the same + // range descriptor to be stored in both meta1 and meta2. + // This happens when some range spans both the meta and the + // user keyspace. Consider when r1 is + // [/Min, /System/NodeLiveness); we'll store the range + // descriptor in both /Meta2/ and in + // /Meta1/KeyMax[1]. // // As part of iterator we'll de-duplicate this descriptor // away by checking whether we've seen it before in meta1. @@ -111,15 +147,34 @@ func (i *iteratorImpl) Iterate( continue } + if _, err := desc.KeySpan().Intersect(rspan); err != nil { + // We're past the last range descriptor that overlaps + // with the given span. + stopMetaIteration = true + break + } + + // This descriptor's span intersects with our query span, so + // collect it for the callback. descriptors = append(descriptors, desc) + if keys.InMeta1(keys.RangeMetaKey(desc.StartKey)) { lastRangeIDInMeta1 = desc.RangeID } } - // Invoke fn with the current chunk (of size ~blockSize) of - // range descriptors. - return fn(descriptors...) - }) + if len(descriptors) != 0 { + // Invoke fn with the current chunk (of size ~pageSize) of + // range descriptors. + if err := fn(descriptors...); err != nil { + return err + } + } + if stopMetaIteration { + return iterutil.StopIteration() // we're done here + } + return nil + }), + ) }) } diff --git a/pkg/util/rangedesciter/rangedesciter_test.go b/pkg/util/rangedesciter/rangedesciter_test.go index 67f79259c109..1f33d6a35b21 100644 --- a/pkg/util/rangedesciter/rangedesciter_test.go +++ b/pkg/util/rangedesciter/rangedesciter_test.go @@ -13,35 +13,64 @@ package rangedesciter_test import ( "context" "fmt" + "strings" "testing" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/tests" + "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/rangedesciter" + "github.com/cockroachdb/datadriven" + "github.com/stretchr/testify/require" ) -func TestIterator(t *testing.T) { +var splits = [][]roachpb.Key{ + {}, // no splits + {keys.Meta2Prefix}, // split between meta1 and meta2 + {keys.SystemPrefix}, // split after the meta range + {keys.Meta2Prefix, keys.SystemPrefix}, // split before and after meta2 + {keys.RangeMetaKey(roachpb.RKey("middle")).AsRawKey()}, // split within meta2 + {keys.Meta2Prefix, keys.RangeMetaKey(roachpb.RKey("middle")).AsRawKey()}, // split at start of and within meta2 +} + +var scopes = []roachpb.Span{ + keys.EverythingSpan, // = /M{in-ax} + keys.NodeLivenessSpan, // = /System/NodeLiveness{-Max} + keys.TimeseriesSpan, // = /System{/tsd-tse} + keys.Meta1Span, // = /M{in-eta2/} + { // = /{Meta1/-System} + Key: keys.MetaMin, + EndKey: keys.MetaMax, + }, + { // = /Table/{3-6} + Key: keys.SystemDescriptorTableSpan.Key, + EndKey: keys.SystemZonesTableSpan.EndKey, + }, + { // = /Table/{38-48} + Key: keys.SystemSQLCodec.TablePrefix(keys.TenantsRangesID), + EndKey: keys.SystemSQLCodec.TablePrefix(keys.SpanConfigurationsTableID + 1), + }, + { // = /Table/{0-Max} + Key: keys.TableDataMin, + EndKey: keys.TableDataMax, + }, +} + +func TestEverythingIterator(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() - for _, splits := range [][]roachpb.Key{ - {}, // no splits - {keys.Meta2Prefix}, // split between meta1 and meta2 - {keys.SystemPrefix}, // split after the meta range - {keys.Meta2Prefix, keys.SystemPrefix}, // split before and after meta2 - {keys.RangeMetaKey(roachpb.RKey("middle")).AsRawKey()}, // split within meta2 - {keys.Meta2Prefix, keys.RangeMetaKey(roachpb.RKey("middle")).AsRawKey()}, // split at start of and within meta2 - } { - t.Run(fmt.Sprintf("with-splits-at=%s", splits), func(t *testing.T) { + for _, s := range splits { + t.Run(fmt.Sprintf("with-splits-at=%s", s), func(t *testing.T) { params, _ := tests.CreateTestServerParams() server, _, kvDB := serverutils.StartServer(t, params) defer server.Stopper().Stop(context.Background()) - for _, split := range splits { + for _, split := range s { if _, _, err := server.SplitRange(split); err != nil { t.Fatal(err) } @@ -56,13 +85,14 @@ func TestIterator(t *testing.T) { } iter := rangedesciter.New(kvDB) - for _, blockSize := range []int{1, 5, 10, 50} { + for _, pageSize := range []int{1, 5, 10, 50} { var numDescs int init := func() { numDescs = 0 } - if err := iter.Iterate(ctx, blockSize, init, func(descriptors ...roachpb.RangeDescriptor) error { - numDescs += len(descriptors) - return nil - }); err != nil { + if err := iter.Iterate(ctx, pageSize, init, keys.EverythingSpan, + func(descriptors ...roachpb.RangeDescriptor) error { + numDescs += len(descriptors) + return nil + }); err != nil { t.Fatal(err) } @@ -73,3 +103,78 @@ func TestIterator(t *testing.T) { }) } } + +// TestDataDriven is a data-driven test for rangedesciter. The following syntax +// is provided: +// +// - "iter" [page-size=] [scope=] +// - "split" [set=] +func TestDataDriven(t *testing.T) { + defer leaktest.AfterTest(t)() + + datadriven.Walk(t, testutils.TestDataPath(t), func(t *testing.T, path string) { + ctx := context.Background() + params, _ := tests.CreateTestServerParams() + server, _, kvDB := serverutils.StartServer(t, params) + defer server.Stopper().Stop(context.Background()) + + iter := rangedesciter.New(kvDB) + datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string { + var buf strings.Builder + + switch d.Cmd { + case "iter": + pageSize := 1 + if d.HasArg("page-size") { + d.ScanArgs(t, "page-size", &pageSize) + } + var scopeIdx int + if d.HasArg("scope") { + d.ScanArgs(t, "scope", &scopeIdx) + require.True(t, scopeIdx >= 0 && scopeIdx < len(scopes)) + } + scope := scopes[scopeIdx] + + var numDescs int + init := func() { numDescs = 0 } + if err := iter.Iterate(ctx, pageSize, init, scope, + func(descriptors ...roachpb.RangeDescriptor) error { + for _, desc := range descriptors { + buf.WriteString(fmt.Sprintf("- r%d:%s\n", desc.RangeID, desc.KeySpan().String())) + } + numDescs += len(descriptors) + return nil + }); err != nil { + t.Fatal(err) + } + + var numRanges int + if err := server.GetStores().(*kvserver.Stores).VisitStores(func(s *kvserver.Store) error { + numRanges = s.ReplicaCount() + return nil + }); err != nil { + t.Fatal(err) + } + buf.WriteString(fmt.Sprintf( + "iteration through %s (page-size=%d) found %d/%d descriptors\n", + scope, pageSize, numDescs, numRanges)) + + case "split": + var set int + d.ScanArgs(t, "set", &set) + require.True(t, set >= 0 && set < len(splits)) + for _, split := range splits[set] { + buf.WriteString(fmt.Sprintf("splitting at %s\n", split)) + if _, _, err := server.SplitRange(split); err != nil { + t.Fatal(err) + } + } + + default: + return fmt.Sprintf("unknown command: %s", d.Cmd) + } + + return buf.String() + }) + }) +} diff --git a/pkg/util/rangedesciter/testdata/scoped_iteration b/pkg/util/rangedesciter/testdata/scoped_iteration new file mode 100644 index 000000000000..ca0825fa4cc2 --- /dev/null +++ b/pkg/util/rangedesciter/testdata/scoped_iteration @@ -0,0 +1,149 @@ +iter +---- +- r1:/{Meta1/-System/NodeLiveness} +- r2:/System/NodeLiveness{-Max} +- r3:/System/{NodeLivenessMax-tsd} +- r4:/System{/tsd-tse} +- r5:/{Systemtse-Table/0} +- r6:/Table/{0-3} +- r7:/Table/{3-4} +- r8:/Table/{4-5} +- r9:/Table/{5-6} +- r10:/Table/{6-7} +- r11:/Table/{7-8} +- r12:/Table/{8-11} +- r13:/Table/1{1-2} +- r14:/Table/1{2-3} +- r15:/Table/1{3-4} +- r16:/Table/1{4-5} +- r17:/Table/1{5-6} +- r18:/Table/1{6-7} +- r19:/Table/1{7-8} +- r20:/Table/1{8-9} +- r21:/Table/{19-20} +- r22:/Table/2{0-1} +- r23:/Table/2{1-2} +- r24:/Table/2{2-3} +- r25:/Table/2{3-4} +- r26:/Table/2{4-5} +- r27:/Table/2{5-6} +- r28:/Table/2{6-7} +- r29:/Table/2{7-8} +- r30:/Table/2{8-9} +- r31:/{Table/29-NamespaceTable/30} +- r32:/NamespaceTable/{30-Max} +- r33:/{NamespaceTable/Max-Table/32} +- r34:/Table/3{2-3} +- r35:/Table/3{3-4} +- r36:/Table/3{4-5} +- r37:/Table/3{5-6} +- r38:/Table/3{6-7} +- r39:/Table/3{7-8} +- r40:/Table/3{8-9} +- r41:/Table/{39-40} +- r42:/Table/4{0-1} +- r43:/Table/4{1-2} +- r44:/Table/4{2-3} +- r45:/Table/4{3-4} +- r46:/Table/4{4-5} +- r47:/Table/4{5-6} +- r48:/Table/4{6-7} +- r49:/Table/4{7-8} +- r50:/Table/{48-50} +- r51:/Table/5{0-1} +- r52:/Table/5{1-2} +- r53:/{Table/52-Max} +iteration through /M{in-ax} (page-size=1) found 53/53 descriptors + +iter scope=1 +---- +- r2:/System/NodeLiveness{-Max} +iteration through /System/NodeLiveness{-Max} (page-size=1) found 1/53 descriptors + +iter scope=2 +---- +- r4:/System{/tsd-tse} +iteration through /System{/tsd-tse} (page-size=1) found 1/53 descriptors + +iter scope=3 +---- +- r1:/{Meta1/-System/NodeLiveness} +iteration through /M{in-eta2/} (page-size=1) found 1/53 descriptors + +iter scope=4 +---- +- r1:/{Meta1/-System/NodeLiveness} +iteration through /{Meta1/-System} (page-size=1) found 1/53 descriptors + +iter scope=5 +---- +- r7:/Table/{3-4} +- r8:/Table/{4-5} +- r9:/Table/{5-6} +iteration through /Table/{3-6} (page-size=1) found 3/53 descriptors + +iter scope=6 +---- +- r40:/Table/3{8-9} +- r41:/Table/{39-40} +- r42:/Table/4{0-1} +- r43:/Table/4{1-2} +- r44:/Table/4{2-3} +- r45:/Table/4{3-4} +- r46:/Table/4{4-5} +- r47:/Table/4{5-6} +- r48:/Table/4{6-7} +- r49:/Table/4{7-8} +iteration through /Table/{38-48} (page-size=1) found 10/53 descriptors + +iter scope=7 +---- +- r6:/Table/{0-3} +- r7:/Table/{3-4} +- r8:/Table/{4-5} +- r9:/Table/{5-6} +- r10:/Table/{6-7} +- r11:/Table/{7-8} +- r12:/Table/{8-11} +- r13:/Table/1{1-2} +- r14:/Table/1{2-3} +- r15:/Table/1{3-4} +- r16:/Table/1{4-5} +- r17:/Table/1{5-6} +- r18:/Table/1{6-7} +- r19:/Table/1{7-8} +- r20:/Table/1{8-9} +- r21:/Table/{19-20} +- r22:/Table/2{0-1} +- r23:/Table/2{1-2} +- r24:/Table/2{2-3} +- r25:/Table/2{3-4} +- r26:/Table/2{4-5} +- r27:/Table/2{5-6} +- r28:/Table/2{6-7} +- r29:/Table/2{7-8} +- r30:/Table/2{8-9} +- r31:/{Table/29-NamespaceTable/30} +- r32:/NamespaceTable/{30-Max} +- r33:/{NamespaceTable/Max-Table/32} +- r34:/Table/3{2-3} +- r35:/Table/3{3-4} +- r36:/Table/3{4-5} +- r37:/Table/3{5-6} +- r38:/Table/3{6-7} +- r39:/Table/3{7-8} +- r40:/Table/3{8-9} +- r41:/Table/{39-40} +- r42:/Table/4{0-1} +- r43:/Table/4{1-2} +- r44:/Table/4{2-3} +- r45:/Table/4{3-4} +- r46:/Table/4{4-5} +- r47:/Table/4{5-6} +- r48:/Table/4{6-7} +- r49:/Table/4{7-8} +- r50:/Table/{48-50} +- r51:/Table/5{0-1} +- r52:/Table/5{1-2} +- r53:/{Table/52-Max} +iteration through /Table/{0-Max} (page-size=1) found 48/53 descriptors diff --git a/pkg/util/rangedesciter/testdata/scoped_iteration_with_page_size b/pkg/util/rangedesciter/testdata/scoped_iteration_with_page_size new file mode 100644 index 000000000000..c03f421805f6 --- /dev/null +++ b/pkg/util/rangedesciter/testdata/scoped_iteration_with_page_size @@ -0,0 +1,80 @@ +iter page-size=3 +---- +- r1:/{Meta1/-System/NodeLiveness} +- r2:/System/NodeLiveness{-Max} +- r3:/System/{NodeLivenessMax-tsd} +- r4:/System{/tsd-tse} +- r5:/{Systemtse-Table/0} +- r6:/Table/{0-3} +- r7:/Table/{3-4} +- r8:/Table/{4-5} +- r9:/Table/{5-6} +- r10:/Table/{6-7} +- r11:/Table/{7-8} +- r12:/Table/{8-11} +- r13:/Table/1{1-2} +- r14:/Table/1{2-3} +- r15:/Table/1{3-4} +- r16:/Table/1{4-5} +- r17:/Table/1{5-6} +- r18:/Table/1{6-7} +- r19:/Table/1{7-8} +- r20:/Table/1{8-9} +- r21:/Table/{19-20} +- r22:/Table/2{0-1} +- r23:/Table/2{1-2} +- r24:/Table/2{2-3} +- r25:/Table/2{3-4} +- r26:/Table/2{4-5} +- r27:/Table/2{5-6} +- r28:/Table/2{6-7} +- r29:/Table/2{7-8} +- r30:/Table/2{8-9} +- r31:/{Table/29-NamespaceTable/30} +- r32:/NamespaceTable/{30-Max} +- r33:/{NamespaceTable/Max-Table/32} +- r34:/Table/3{2-3} +- r35:/Table/3{3-4} +- r36:/Table/3{4-5} +- r37:/Table/3{5-6} +- r38:/Table/3{6-7} +- r39:/Table/3{7-8} +- r40:/Table/3{8-9} +- r41:/Table/{39-40} +- r42:/Table/4{0-1} +- r43:/Table/4{1-2} +- r44:/Table/4{2-3} +- r45:/Table/4{3-4} +- r46:/Table/4{4-5} +- r47:/Table/4{5-6} +- r48:/Table/4{6-7} +- r49:/Table/4{7-8} +- r50:/Table/{48-50} +- r51:/Table/5{0-1} +- r52:/Table/5{1-2} +- r53:/{Table/52-Max} +iteration through /M{in-ax} (page-size=3) found 53/53 descriptors + +iter scope=1 page-size=5 +---- +- r2:/System/NodeLiveness{-Max} +iteration through /System/NodeLiveness{-Max} (page-size=5) found 1/53 descriptors + +iter scope=1 page-size=500 +---- +- r2:/System/NodeLiveness{-Max} +iteration through /System/NodeLiveness{-Max} (page-size=500) found 1/53 descriptors + +iter scope=6 page-size=4 +---- +- r40:/Table/3{8-9} +- r41:/Table/{39-40} +- r42:/Table/4{0-1} +- r43:/Table/4{1-2} +- r44:/Table/4{2-3} +- r45:/Table/4{3-4} +- r46:/Table/4{4-5} +- r47:/Table/4{5-6} +- r48:/Table/4{6-7} +- r49:/Table/4{7-8} +iteration through /Table/{38-48} (page-size=4) found 10/53 descriptors diff --git a/pkg/util/rangedesciter/testdata/scoped_iteration_with_splits b/pkg/util/rangedesciter/testdata/scoped_iteration_with_splits new file mode 100644 index 000000000000..12706aa2cbaa --- /dev/null +++ b/pkg/util/rangedesciter/testdata/scoped_iteration_with_splits @@ -0,0 +1,78 @@ +# Split at the start of meta2 and somewhere within meta2 itself. +split set=5 +---- +splitting at /Meta2/"" +splitting at /Meta2/"middle" + +iter +---- +- r1:/Meta{1/-2/} +- r54:/Meta2/{-middle} +- r55:/{Meta2/middle-System/NodeLiveness} +- r2:/System/NodeLiveness{-Max} +- r3:/System/{NodeLivenessMax-tsd} +- r4:/System{/tsd-tse} +- r5:/{Systemtse-Table/0} +- r6:/Table/{0-3} +- r7:/Table/{3-4} +- r8:/Table/{4-5} +- r9:/Table/{5-6} +- r10:/Table/{6-7} +- r11:/Table/{7-8} +- r12:/Table/{8-11} +- r13:/Table/1{1-2} +- r14:/Table/1{2-3} +- r15:/Table/1{3-4} +- r16:/Table/1{4-5} +- r17:/Table/1{5-6} +- r18:/Table/1{6-7} +- r19:/Table/1{7-8} +- r20:/Table/1{8-9} +- r21:/Table/{19-20} +- r22:/Table/2{0-1} +- r23:/Table/2{1-2} +- r24:/Table/2{2-3} +- r25:/Table/2{3-4} +- r26:/Table/2{4-5} +- r27:/Table/2{5-6} +- r28:/Table/2{6-7} +- r29:/Table/2{7-8} +- r30:/Table/2{8-9} +- r31:/{Table/29-NamespaceTable/30} +- r32:/NamespaceTable/{30-Max} +- r33:/{NamespaceTable/Max-Table/32} +- r34:/Table/3{2-3} +- r35:/Table/3{3-4} +- r36:/Table/3{4-5} +- r37:/Table/3{5-6} +- r38:/Table/3{6-7} +- r39:/Table/3{7-8} +- r40:/Table/3{8-9} +- r41:/Table/{39-40} +- r42:/Table/4{0-1} +- r43:/Table/4{1-2} +- r44:/Table/4{2-3} +- r45:/Table/4{3-4} +- r46:/Table/4{4-5} +- r47:/Table/4{5-6} +- r48:/Table/4{6-7} +- r49:/Table/4{7-8} +- r50:/Table/{48-50} +- r51:/Table/5{0-1} +- r52:/Table/5{1-2} +- r53:/{Table/52-Max} +iteration through /M{in-ax} (page-size=1) found 55/55 descriptors + +# Scanning through just meta1 is unnaffected by these splits. +iter scope=3 +---- +- r1:/Meta{1/-2/} +iteration through /M{in-eta2/} (page-size=1) found 1/55 descriptors + +# Scanning through both meta{1,2} does surface these descriptors. +iter scope=4 page-size=2 +---- +- r1:/Meta{1/-2/} +- r54:/Meta2/{-middle} +- r55:/{Meta2/middle-System/NodeLiveness} +iteration through /{Meta1/-System} (page-size=2) found 3/55 descriptors From b19f6b1085821121252375f4598f251249dc1b74 Mon Sep 17 00:00:00 2001 From: GirishV Date: Mon, 21 Nov 2022 17:59:20 +0530 Subject: [PATCH 2/2] sql/sem: add support for "pg_blocking_pids" builtin This is needed for pgadmin. Release note (sql change): Added support for pg_blocking_pids builtin function. It is hardcoded to return an empty array because CockroachDB has no equivalent concept of PIDs as in Postgres. sql/sem: added field for pg_blocking_pids in builtinOidsBySignature map --- pkg/sql/sem/builtins/fixed_oids.go | 1 + pkg/sql/sem/builtins/pg_builtins.go | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/pkg/sql/sem/builtins/fixed_oids.go b/pkg/sql/sem/builtins/fixed_oids.go index 6b339ab1312d..205b0828f4c1 100644 --- a/pkg/sql/sem/builtins/fixed_oids.go +++ b/pkg/sql/sem/builtins/fixed_oids.go @@ -1204,6 +1204,7 @@ var builtinOidsBySignature = map[string]oid.Oid{ `percentile_disc_impl(arg1: float[], arg2: varbit) -> varbit[]`: 302, `pg_advisory_unlock(int: int) -> bool`: 1428, `pg_backend_pid() -> int`: 1400, + `pg_blocking_pids() -> int[]`: 2045, `pg_client_encoding() -> string`: 1429, `pg_collation_for(str: anyelement) -> string`: 1418, `pg_column_is_updatable(reloid: oid, attnum: int2, include_triggers: bool) -> bool`: 1434, diff --git a/pkg/sql/sem/builtins/pg_builtins.go b/pkg/sql/sem/builtins/pg_builtins.go index 20de02ab69fa..1f20c87fc99f 100644 --- a/pkg/sql/sem/builtins/pg_builtins.go +++ b/pkg/sql/sem/builtins/pg_builtins.go @@ -1933,6 +1933,18 @@ SELECT description }, ), + "pg_blocking_pids": makeBuiltin(defProps(), + tree.Overload{ + Types: tree.ArgTypes{}, + ReturnType: tree.FixedReturnType(types.IntArray), + Fn: func(_ context.Context, _ *eval.Context, args tree.Datums) (tree.Datum, error) { + return tree.NewDArray(types.Int), nil + }, + Info: notUsableInfo, + Volatility: volatility.Stable, + }, + ), + // pg_column_size(any) - number of bytes used to store a particular value // (possibly compressed)