Continuously attempt to unseal if sealed keys are supported

command/server.go

@@ -741,7 +741,7 @@ CLUSTER_SYNTHESIS_COMPLETE:
 	// Initialize the core
 	core, newCoreError := vault.NewCore(coreConfig)
 	if newCoreError != nil {
-		if !errwrap.ContainsType(newCoreError, new(vault.NonFatalError)) {
+		if vault.IsFatalError(newCoreError) {
 			c.UI.Error(fmt.Sprintf("Error initializing core: %s", newCoreError))
 			return 1
 		}
@@ -937,12 +937,31 @@ CLUSTER_SYNTHESIS_COMPLETE:
 		return 1
 	}
 
-	if err := core.UnsealWithStoredKeys(context.Background()); err != nil {
-		if !errwrap.ContainsType(err, new(vault.NonFatalError)) {
-			c.UI.Error(fmt.Sprintf("Error initializing core: %s", err))
-			return 1
+	// Attempt unsealing in a background goroutine. This is needed for when a
+	// Vault cluster with multiple servers is configured with auto-unseal but is
+	// uninitialized. Once one server initializes the storage backend, this
+	// goroutine will pick up the unseal keys and unseal this instance.
+	go func() {
+		for {
+			err := core.UnsealWithStoredKeys(context.Background())
+			if err == nil {
+				return
+			}
+
+			if vault.IsFatalError(err) {
+				c.logger.Error(fmt.Sprintf("Error unsealing core", "error", err))
+				return
+			} else {
+				c.logger.Warn(fmt.Sprintf("Failed to unseal core", "error" err))
+			}
+
+			select {
+			case <-c.ShutdownCh:
+				return
+			case <-time.After(5 * time.Second):
+			}
 		}
-	}
+	}()
 
 	// Perform service discovery registrations and initialization of
 	// HTTP server after the verifyOnly check.

http/sys_init.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/vault/vault"
 )
 
@@ -104,7 +103,7 @@ func handleSysInitPut(core *vault.Core, w http.ResponseWriter, r *http.Request)
 
 	result, initErr := core.Initialize(ctx, initParams)
 	if initErr != nil {
-		if !errwrap.ContainsType(initErr, new(vault.NonFatalError)) {
+		if vault.IsFatalError(initErr) {
 			respondError(w, http.StatusBadRequest, initErr)
 			return
 		} else {
@@ -136,7 +135,10 @@ func handleSysInitPut(core *vault.Core, w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	core.UnsealWithStoredKeys(ctx)
+	if err := core.UnsealWithStoredKeys(ctx); err != nil {
+		respondError(w, http.StatusInternalServerError, err)
+		return
+	}
 
 	respondOk(w, resp)
 }

vault/core.go

@@ -107,6 +107,16 @@ func (e *NonFatalError) Error() string {
 	return e.Err.Error()
 }
 
+// NewNonFatalError returns a new non-fatal error.
+func NewNonFatalError(err error) *NonFatalError {
+	return &NonFatalError{Err: err}
+}
+
+// IsFatalError returns true if the given error is a non-fatal error.
+func IsFatalError(err error) bool {
+	return !errwrap.ContainsType(err, new(NonFatalError))
+}
+
 // ErrInvalidKey is returned if there is a user-based error with a provided
 // unseal key. This will be shown to the user, so should not contain
 // information that is sensitive.
@@ -386,6 +396,10 @@ type Core struct {
 	// Stores the sealunwrapper for downgrade needs
 	sealUnwrapper physical.Backend
 
+	// unsealwithStoredKeysLock is a mutex that prevents multiple processes from
+	// unsealing with stored keys are the same time.
+	unsealWithStoredKeysLock sync.Mutex
+
 	// Stores any funcs that should be run on successful postUnseal
 	postUnsealFuncs []func()
 

vault/init.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 
 	"github.com/hashicorp/errwrap"
@@ -270,54 +271,63 @@ func (c *Core) Initialize(ctx context.Context, initParams *InitParams) (*InitRes
 	return results, nil
 }
 
-// UnsealWithStoredKeys performs auto-unseal using stored keys.
+// UnsealWithStoredKeys performs auto-unseal using stored keys. An error
+// return value of "nil" implies the Vault instance is unsealed.
+//
+// Callers should attempt to retry any NOnFatalErrors. Callers should
+// not re-attempt fatal errors.
 func (c *Core) UnsealWithStoredKeys(ctx context.Context) error {
+	c.unsealWithStoredKeysLock.Lock()
+	defer c.unsealWithStoredKeysLock.Unlock()
+
 	if !c.seal.StoredKeysSupported() {
 		return nil
 	}
 
 	// Disallow auto-unsealing when migrating
 	if c.IsInSealMigration() {
-		return nil
+		return NewNonFatalError(errors.New("cannot auto-unseal during seal migration"))
 	}
 
 	sealed := c.Sealed()
 	if !sealed {
+		c.Logger().Warn("attempted unseal with stored keys, but vault is already unsealed")
 		return nil
 	}
 
-	c.logger.Info("stored unseal keys supported, attempting fetch")
+	c.Logger().Info("stored unseal keys supported, attempting fetch")
 	keys, err := c.seal.GetStoredKeys(ctx)
 	if err != nil {
-		c.logger.Error("fetching stored unseal keys failed", "error", err)
-		return &NonFatalError{Err: errwrap.Wrapf("fetching stored unseal keys failed: {{err}}", err)}
+		return NewNonFatalError(errwrap.Wrapf("fetching stored unseal keys failed: {{err}}", err))
 	}
+
+	// This usually happens when auto-unseal is configured, but the servers have
+	// not been initialized yet.
 	if len(keys) == 0 {
-		c.logger.Warn("stored unseal key(s) supported but none found")
-	} else {
-		unsealed := false
-		keysUsed := 0
-		for _, key := range keys {
-			unsealed, err = c.Unseal(key)
-			if err != nil {
-				c.logger.Error("unseal with stored unseal key failed", "error", err)
-				return &NonFatalError{Err: errwrap.Wrapf("unseal with stored key failed: {{err}}", err)}
-			}
-			keysUsed += 1
-			if unsealed {
-				break
-			}
+		return NewNonFatalError(errors.New("stored unseal keys are supported, but none were found"))
+	}
+
+	unsealed := false
+	keysUsed := 0
+	for _, key := range keys {
+		unsealed, err = c.Unseal(key)
+		if err != nil {
+			return NewNonFatalError(errwrap.Wrapf("unseal with stored key failed: {{err}}", err))
 		}
-		if !unsealed {
-			if c.logger.IsWarn() {
-				c.logger.Warn("stored unseal key(s) used but Vault not unsealed yet", "stored_keys_used", keysUsed)
-			}
-		} else {
-			if c.logger.IsInfo() {
-				c.logger.Info("successfully unsealed with stored key(s)", "stored_keys_used", keysUsed)
-			}
+		keysUsed++
+		if unsealed {
+			break
 		}
 	}
 
+	if !unsealed {
+		// This most likely means that the user configured Vault to only store a
+		// subset of the required threshold of keys. We still consider this a
+		// "success", since trying again would yield the same result.
+		c.Logger().Warn("vault still sealed after using stored unseal keys", "stored_keys_used", keysUsed)
+	} else {
+		c.Logger().Info("unsealed with stored keys", "stored_keys_used", keysUsed)
+	}
+
 	return nil
 }