Merge cockroachdb#57650 cockroachdb#58036

57650: migration: introduce various other bells and whistles... r=irfansharif a=irfansharif

...to pave the way for real migrations

This PR cannibalizes most of our prototype in cockroachdb#57445 and shimmies in a few
interfaces to make it all unit-testable. While here, we:
- parallelize the execution of the EveryNode primitive
- introduce the IterateRangeDescriptors primitive
- re-write the migration registration process for better ergonomics
- introduce a more structured Migration type, to annotate it with migration
  specific metadata (it also paves the way for logging the progress of an
  ongoing migration to a system table).

See individual commits for details. 


58036: roachtest: add test_9_6_diagnostics to psycopg blocklist r=rafiss a=arulajmani

Previously, this test was skipped as our pg server version (9.5) did
not meet the threshold (9.6) for this test to run. After the pg server
version bump to 13 this is no longer the case. A change to explicitly
skip this test for CRDB has been merged upstream, but it won't apply
until the next release (psycopg_2_8_7) comes out and we switch to
testing against that. Until then, this test lives in the blocklist.

Closes cockroachdb#57986

Release note: None

Co-authored-by: irfan sharif <[email protected]>
Co-authored-by: arulajmani <[email protected]>

pkg/cmd/roachtest/psycopg_blocklist.go

@@ -29,7 +29,8 @@ var psycopgBlocklists = blocklistsForVersion{
 // After a failed run, an updated version of this blocklist should be available
 // in the test log.
 var psycopgBlockList21_1 = blocklist{
-	"tests.test_async_keyword.CancelTests.test_async_cancel": "41335",
+	"tests.test_async_keyword.CancelTests.test_async_cancel":    "41335",
+	"tests.test_module.ExceptionsTestCase.test_9_6_diagnostics": "58035",
 }
 
 var psycopgBlockList20_2 = blocklist{

pkg/migration/BUILD.bazel

@@ -1,8 +1,9 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "migration",
     srcs = [
+        "helper.go",
         "manager.go",
         "migrations.go",
         "util.go",
@@ -11,15 +12,50 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/clusterversion",
+        "//pkg/keys",
         "//pkg/kv",
         "//pkg/kv/kvserver/liveness/livenesspb",
         "//pkg/roachpb",
         "//pkg/rpc",
         "//pkg/rpc/nodedialer",
         "//pkg/server/serverpb",
         "//pkg/sql",
+        "//pkg/sql/sqlutil",
+        "//pkg/util/ctxgroup",
         "//pkg/util/log",
+        "//pkg/util/quotapool",
         "//vendor/github.com/cockroachdb/errors",
         "//vendor/github.com/cockroachdb/logtags",
+        "//vendor/github.com/cockroachdb/redact",
+        "//vendor/google.golang.org/grpc",
+    ],
+)
+
+go_test(
+    name = "migration_test",
+    srcs = [
+        "client_test.go",
+        "helper_test.go",
+        "main_test.go",
+        "util_test.go",
+    ],
+    embed = [":migration"],
+    deps = [
+        "//pkg/clusterversion",
+        "//pkg/kv",
+        "//pkg/kv/kvserver",
+        "//pkg/kv/kvserver/liveness/livenesspb",
+        "//pkg/roachpb",
+        "//pkg/security",
+        "//pkg/security/securitytest",
+        "//pkg/server",
+        "//pkg/server/serverpb",
+        "//pkg/sql/tests",
+        "//pkg/testutils",
+        "//pkg/testutils/serverutils",
+        "//pkg/testutils/testcluster",
+        "//pkg/util/leaktest",
+        "//pkg/util/syncutil",
+        "//vendor/google.golang.org/grpc",
     ],
 )

pkg/migration/client_test.go

@@ -0,0 +1,61 @@
+// Copyright 2020 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+package migration_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/clusterversion"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
+	"github.com/cockroachdb/cockroach/pkg/migration"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/sql/tests"
+	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+)
+
+func TestHelperIterateRangeDescriptors(t *testing.T) {
+	defer leaktest.AfterTest(t)
+
+	cv := clusterversion.ClusterVersion{}
+	ctx := context.Background()
+	const numNodes = 1
+
+	params, _ := tests.CreateTestServerParams()
+	server, _, kvDB := serverutils.StartServer(t, params)
+	defer server.Stopper().Stop(context.Background())
+
+	var numRanges int
+	if err := server.GetStores().(*kvserver.Stores).VisitStores(func(s *kvserver.Store) error {
+		numRanges = s.ReplicaCount()
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	c := migration.TestingNewCluster(numNodes, migration.TestingWithKV(kvDB))
+	h := migration.TestingNewHelper(c, cv)
+
+	for _, blockSize := range []int{1, 5, 10, 50} {
+		var numDescs int
+		init := func() { numDescs = 0 }
+		if err := h.IterateRangeDescriptors(ctx, blockSize, init, func(descriptors ...roachpb.RangeDescriptor) error {
+			numDescs += len(descriptors)
+			return nil
+		}); err != nil {
+			t.Fatal(err)
+		}
+
+		if numDescs != numRanges {
+			t.Fatalf("expected to find %d ranges, found %d", numRanges+1, numDescs)
+		}
+	}
+}