Test Shamir-to-Transit and Transit-to-Shamir Seal Migration for post-1.4 Vault.

helper/testhelpers/testhelpers.go

@@ -412,16 +412,9 @@ func (p *TestRaftServerAddressProvider) ServerAddr(id raftlib.ServerID) (raftlib
 }
 
 func RaftClusterJoinNodes(t testing.T, cluster *vault.TestCluster) {
-	raftClusterJoinNodes(t, cluster, false)
-}
-
-func RaftClusterJoinNodesWithStoredKeys(t testing.T, cluster *vault.TestCluster) {
-	raftClusterJoinNodes(t, cluster, true)
-}
-
-func raftClusterJoinNodes(t testing.T, cluster *vault.TestCluster, useStoredKeys bool) {
 
 	addressProvider := &TestRaftServerAddressProvider{Cluster: cluster}
+
 	atomic.StoreUint32(&vault.UpdateClusterAddrForTests, 1)
 
 	leader := cluster.Cores[0]
@@ -430,11 +423,7 @@ func raftClusterJoinNodes(t testing.T, cluster *vault.TestCluster, useStoredKeys
 	{
 		EnsureCoreSealed(t, leader)
 		leader.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(addressProvider)
-		if useStoredKeys {
-			cluster.UnsealCoreWithStoredKeys(t, leader)
-		} else {
-			cluster.UnsealCore(t, leader)
-		}
+		cluster.UnsealCore(t, leader)
 		vault.TestWaitActive(t, leader.Core)
 	}
 
@@ -454,6 +443,37 @@ func raftClusterJoinNodes(t testing.T, cluster *vault.TestCluster, useStoredKeys
 			t.Fatal(err)
 		}
 
+		cluster.UnsealCore(t, core)
+	}
+
+	WaitForNCoresUnsealed(t, cluster, len(cluster.Cores))
+}
+
+// JoinRaftFollowers unseals the leader, and then joins-and-unseals the
+// followers one at a time.  We assume that the ServerAddressProvider has
+// already been installed on all the nodes.
+func JoinRaftFollowers(t testing.T, cluster *vault.TestCluster, useStoredKeys bool) {
+
+	leader := cluster.Cores[0]
+
+	cluster.UnsealCore(t, leader)
+	vault.TestWaitActive(t, leader.Core)
+
+	leaderInfos := []*raft.LeaderJoinInfo{
+		&raft.LeaderJoinInfo{
+			LeaderAPIAddr: leader.Client.Address(),
+			TLSConfig:     leader.TLSConfig,
+		},
+	}
+
+	// Join followers
+	for i := 1; i < len(cluster.Cores); i++ {
+		core := cluster.Cores[i]
+		_, err := core.JoinRaftCluster(namespace.RootContext(context.Background()), leaderInfos, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		if useStoredKeys {
 			// For autounseal, the raft backend is not initialized right away
 			// after the join.  We need to wait briefly before we can unseal.
@@ -505,11 +525,11 @@ func (p *HardcodedServerAddressProvider) ServerAddr(id raftlib.ServerID) (raftli
 
 // NewHardcodedServerAddressProvider is a convenience function that makes a
 // ServerAddressProvider from a given cluster address base port.
-func NewHardcodedServerAddressProvider(cluster *vault.TestCluster, baseClusterPort int) raftlib.ServerAddressProvider {
+func NewHardcodedServerAddressProvider(numCores, baseClusterPort int) raftlib.ServerAddressProvider {
 
 	entries := make(map[raftlib.ServerID]raftlib.ServerAddress)
 
-	for i := 0; i < len(cluster.Cores); i++ {
+	for i := 0; i < numCores; i++ {
 		id := fmt.Sprintf("core-%d", i)
 		addr := fmt.Sprintf("127.0.0.1:%d", baseClusterPort+i)
 		entries[raftlib.ServerID(id)] = raftlib.ServerAddress(addr)
@@ -520,17 +540,6 @@ func NewHardcodedServerAddressProvider(cluster *vault.TestCluster, baseClusterPo
 	}
 }
 
-// SetRaftAddressProviders sets a ServerAddressProvider for all the nodes in a
-// cluster.
-func SetRaftAddressProviders(t testing.T, cluster *vault.TestCluster, provider raftlib.ServerAddressProvider) {
-
-	atomic.StoreUint32(&vault.UpdateClusterAddrForTests, 1)
-
-	for _, core := range cluster.Cores {
-		core.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(provider)
-	}
-}
-
 // VerifyRaftConfiguration checks that we have a valid raft configuration, i.e.
 // the correct number of servers, having the correct NodeIDs, and exactly one
 // leader.
@@ -565,6 +574,35 @@ func VerifyRaftConfiguration(core *vault.TestClusterCore, numCores int) error {
 	return nil
 }
 
+// AwaitLeader waits for one of the cluster's nodes to become leader.
+func AwaitLeader(t testing.T, cluster *vault.TestCluster) (int, error) {
+
+	timeout := time.Now().Add(30 * time.Second)
+	for {
+		if time.Now().After(timeout) {
+			break
+		}
+
+		for i, core := range cluster.Cores {
+			if core.Core.Sealed() {
+				continue
+			}
+
+			isLeader, _, _, err := core.Leader()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if isLeader {
+				return i, nil
+			}
+		}
+
+		time.Sleep(time.Second)
+	}
+
+	return 0, fmt.Errorf("timeout waiting leader")
+}
+
 func GenerateDebugLogs(t testing.T, client *api.Client) chan struct{} {
 	t.Helper()
 

helper/testhelpers/teststorage/teststorage_reusable.go

@@ -8,6 +8,7 @@ import (
 	"github.com/mitchellh/go-testing-interface"
 
 	hclog "github.com/hashicorp/go-hclog"
+	raftlib "github.com/hashicorp/raft"
 	"github.com/hashicorp/vault/physical/raft"
 	"github.com/hashicorp/vault/vault"
 )
@@ -73,7 +74,10 @@ func MakeReusableStorage(t testing.T, logger hclog.Logger, bundle *vault.Physica
 
 // MakeReusableRaftStorage makes a physical raft backend that can be re-used
 // across multiple test clusters in sequence.
-func MakeReusableRaftStorage(t testing.T, logger hclog.Logger, numCores int) (ReusableStorage, StorageCleanup) {
+func MakeReusableRaftStorage(
+	t testing.T, logger hclog.Logger, numCores int,
+	addressProvider raftlib.ServerAddressProvider,
+) (ReusableStorage, StorageCleanup) {
 
 	raftDirs := make([]string, numCores)
 	for i := 0; i < numCores; i++ {
@@ -87,17 +91,14 @@ func MakeReusableRaftStorage(t testing.T, logger hclog.Logger, numCores int) (Re
 			conf.DisablePerformanceStandby = true
 			opts.KeepStandbysSealed = true
 			opts.PhysicalFactory = func(t testing.T, coreIdx int, logger hclog.Logger) *vault.PhysicalBackendBundle {
-				return makeReusableRaftBackend(t, coreIdx, logger, raftDirs[coreIdx])
+				return makeReusableRaftBackend(t, coreIdx, logger, raftDirs[coreIdx], addressProvider)
 			}
 		},
 
 		// Close open files being used by raft.
 		Cleanup: func(t testing.T, cluster *vault.TestCluster) {
-			for _, core := range cluster.Cores {
-				raftStorage := core.UnderlyingRawStorage.(*raft.RaftBackend)
-				if err := raftStorage.Close(); err != nil {
-					t.Fatal(err)
-				}
+			for i := 0; i < len(cluster.Cores); i++ {
+				CloseRaftStorage(t, cluster, i)
 			}
 		},
 	}
@@ -111,6 +112,14 @@ func MakeReusableRaftStorage(t testing.T, logger hclog.Logger, numCores int) (Re
 	return storage, cleanup
 }
 
+// CloseRaftStorage closes open files being used by raft.
+func CloseRaftStorage(t testing.T, cluster *vault.TestCluster, idx int) {
+	raftStorage := cluster.Cores[idx].UnderlyingRawStorage.(*raft.RaftBackend)
+	if err := raftStorage.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func makeRaftDir(t testing.T) string {
 	raftDir, err := ioutil.TempDir("", "vault-raft-")
 	if err != nil {
@@ -120,7 +129,10 @@ func makeRaftDir(t testing.T) string {
 	return raftDir
 }
 
-func makeReusableRaftBackend(t testing.T, coreIdx int, logger hclog.Logger, raftDir string) *vault.PhysicalBackendBundle {
+func makeReusableRaftBackend(
+	t testing.T, coreIdx int, logger hclog.Logger, raftDir string,
+	addressProvider raftlib.ServerAddressProvider,
+) *vault.PhysicalBackendBundle {
 
 	nodeID := fmt.Sprintf("core-%d", coreIdx)
 	conf := map[string]string{
@@ -134,6 +146,8 @@ func makeReusableRaftBackend(t testing.T, coreIdx int, logger hclog.Logger, raft
 		t.Fatal(err)
 	}
 
+	backend.(*raft.RaftBackend).SetServerAddressProvider(addressProvider)
+
 	return &vault.PhysicalBackendBundle{
 		Backend: backend,
 	}

vault/external_tests/sealmigration/seal_migration_pre14_test.go

@@ -25,7 +25,7 @@ import (
 func TestSealMigration_TransitToShamir_Pre14(t *testing.T) {
 	// Note that we do not test integrated raft storage since this is
 	// a pre-1.4 test.
-	testVariousBackends(t, testSealMigrationTransitToShamir_Pre14, false)
+	testVariousBackends(t, testSealMigrationTransitToShamir_Pre14, basePort_TransitToShamir_Pre14, false)
 }
 
 func testSealMigrationTransitToShamir_Pre14(
@@ -42,7 +42,11 @@ func testSealMigrationTransitToShamir_Pre14(
 	tss.MakeKey(t, "transit-seal-key")
 
 	// Initialize the backend with transit.
-	rootToken, recoveryKeys, transitSeal := initializeTransit(t, logger, storage, basePort, tss)
+	cluster, _, transitSeal := initializeTransit(t, logger, storage, basePort, tss)
+	rootToken, recoveryKeys := cluster.RootToken, cluster.RecoveryKeys
+	cluster.EnsureCoresSealed(t)
+	storage.Cleanup(t, cluster)
+	cluster.Cleanup()
 
 	// Migrate the backend from transit to shamir
 	migrateFromTransitToShamir_Pre14(t, logger, storage, basePort, tss, transitSeal, rootToken, recoveryKeys)