work on transit join

helper/testhelpers/testhelpers.go

@@ -426,17 +426,17 @@ func RaftClusterJoinNodes(t testing.T, cluster *vault.TestCluster) {
 		vault.TestWaitActive(t, leader.Core)
 	}
 
-	leaderInfo := &raft.LeaderJoinInfo{
-		LeaderAPIAddr: leader.Client.Address(),
-		TLSConfig:     leader.TLSConfig,
+	leaderInfos := []*raft.LeaderJoinInfo{
+		&raft.LeaderJoinInfo{
+			LeaderAPIAddr: leader.Client.Address(),
+			TLSConfig:     leader.TLSConfig,
+		},
 	}
 
+	// Join followers
 	for i := 1; i < len(cluster.Cores); i++ {
 		core := cluster.Cores[i]
 		core.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(addressProvider)
-		leaderInfos := []*raft.LeaderJoinInfo{
-			leaderInfo,
-		}
 		_, err := core.JoinRaftCluster(namespace.RootContext(context.Background()), leaderInfos, false)
 		if err != nil {
 			t.Fatal(err)
@@ -450,8 +450,59 @@ func RaftClusterJoinNodes(t testing.T, cluster *vault.TestCluster) {
 
 func RaftClusterJoinNodesWithStoredKeys(t testing.T, cluster *vault.TestCluster) {
 
+	addressProvider := &TestRaftServerAddressProvider{Cluster: cluster}
+	atomic.StoreUint32(&vault.UpdateClusterAddrForTests, 1)
+
 	leader := cluster.Cores[0]
+
+	// Seal the leader so we can install an address provider
+	{
+		EnsureCoreSealed(t, leader)
+		leader.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(addressProvider)
+		if err := leader.UnsealWithStoredKeys(context.Background()); err != nil {
+			t.Fatal(err)
+		}
+		vault.TestWaitActive(t, leader.Core)
+	}
+
+	leaderInfo := &raft.LeaderJoinInfo{
+		LeaderAPIAddr: leader.Client.Address(),
+		TLSConfig:     leader.TLSConfig,
+	}
+
+	for i := 1; i < len(cluster.Cores); i++ {
+		core := cluster.Cores[i]
+		core.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(addressProvider)
+		leaderInfos := []*raft.LeaderJoinInfo{
+			leaderInfo,
+		}
+		_, err := core.JoinRaftCluster(namespace.RootContext(context.Background()), leaderInfos, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// The raft backend is not initialized right away after the join.  We
+		// need to wait briefly before we can unseal.
+		timeout := time.Now().Add(30 * time.Second)
+		for {
+			if time.Now().After(timeout) {
+				t.Fatal("timeout waiting for core to unseal")
+			}
+			err := core.UnsealWithStoredKeys(context.Background())
+			if err == nil {
+				return
+			}
+			core.Logger().Warn("failed to unseal core", "error", err)
+			time.Sleep(time.Second)
+		}
+	}
+
 	debugRaftConfiguration(t, leader)
+	for i, c := range cluster.Cores {
+		fmt.Printf(">>> core sealed %d %t\n", i, c.Core.Sealed())
+	}
+
+	WaitForNCoresUnsealed(t, cluster, len(cluster.Cores))
 }
 
 // HardcodedServerAddressProvider is a ServerAddressProvider that uses
@@ -544,7 +595,8 @@ func debugRaftConfiguration(t testing.T, core *vault.TestClusterCore) {
 	}
 
 	servers := config.Servers
+	fmt.Printf("--------------------------------------------------------\n")
 	for i, s := range servers {
-		fmt.Printf(">>> debugRaft %d %s %t\n", i, s.NodeID, s.Leader)
+		fmt.Printf(">>> debugRaft %d %q %t\n", i, s.NodeID, s.Leader)
 	}
 }

vault/external_tests/seal_migration/seal_migration_test.go

@@ -257,8 +257,9 @@ func initializeTransit(
 		//if err := testhelpers.VerifyRaftConfiguration(leader, numTestCores); err != nil {
 		//	t.Fatal(err)
 		//}
+	} else {
+		testhelpers.WaitForNCoresUnsealed(t, cluster, numTestCores)
 	}
-	testhelpers.WaitForNCoresUnsealed(t, cluster, numTestCores)
 
 	// Write a secret that we will read back out later.
 	_, err := client.Logical().Write(

vault/init.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"runtime/debug"
 	"sync/atomic"
 
 	wrapping "github.com/hashicorp/go-kms-wrapping"
@@ -416,6 +417,10 @@ func (c *Core) UnsealWithStoredKeys(ctx context.Context) error {
 	c.unsealWithStoredKeysLock.Lock()
 	defer c.unsealWithStoredKeysLock.Unlock()
 
+	fmt.Printf("--------------------------------------------------------------------------\n")
+	fmt.Printf("Core.UnsealWithStoredKeys\n")
+	debug.PrintStack()
+
 	if c.seal.BarrierType() == wrapping.Shamir {
 		return nil
 	}

vault/testing.go

@@ -908,13 +908,6 @@ func (c *TestCluster) UnsealCore(t testing.T, core *TestClusterCore) {
 	}
 }
 
-func (c *TestCluster) UnsealCoreWithStoredKeys(t testing.T, core *TestClusterCore) {
-	err := core.UnsealWithStoredKeys(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
 func (c *TestCluster) EnsureCoresSealed(t testing.T) {
 	t.Helper()
 	if err := c.ensureCoresSealed(); err != nil {