Fix: rotate root credentials for database plugins using WAL

builtin/logical/database/backend.go

@@ -6,6 +6,7 @@ import (
 	"net/rpc"
 	"strings"
 	"sync"
+	"time"
 
 	log "github.com/hashicorp/go-hclog"
 
@@ -24,6 +25,7 @@ const (
 	databaseConfigPath     = "database/config/"
 	databaseRolePath       = "role/"
 	databaseStaticRolePath = "static-role/"
+	minRootCredRollbackAge = 1 * time.Minute
 )
 
 type dbPluginInstance struct {
@@ -93,9 +95,11 @@ func Backend(conf *logical.BackendConfig) *databaseBackend {
 		Secrets: []*framework.Secret{
 			secretCreds(&b),
 		},
-		Clean:       b.clean,
-		Invalidate:  b.invalidate,
-		BackendType: logical.TypeLogical,
+		Clean:             b.clean,
+		Invalidate:        b.invalidate,
+		WALRollback:       b.walRollback,
+		WALRollbackMinAge: minRootCredRollbackAge,
+		BackendType:       logical.TypeLogical,
 	}
 
 	b.logger = conf.Logger
@@ -223,6 +227,15 @@ func (b *databaseBackend) invalidate(ctx context.Context, key string) {
 }
 
 func (b *databaseBackend) GetConnection(ctx context.Context, s logical.Storage, name string) (*dbPluginInstance, error) {
+	config, err := b.DatabaseConfig(ctx, s, name)
+	if err != nil {
+		return nil, err
+	}
+
+	return b.GetConnectionWithConfig(ctx, name, config)
+}
+
+func (b *databaseBackend) GetConnectionWithConfig(ctx context.Context, name string, config *DatabaseConfig) (*dbPluginInstance, error) {
 	b.RLock()
 	unlockFunc := b.RUnlock
 	defer func() { unlockFunc() }()
@@ -242,11 +255,6 @@ func (b *databaseBackend) GetConnection(ctx context.Context, s logical.Storage,
 		return db, nil
 	}
 
-	config, err := b.DatabaseConfig(ctx, s, name)
-	if err != nil {
-		return nil, err
-	}
-
 	dbp, err := dbplugin.PluginFactory(ctx, config.PluginName, b.System(), b.logger)
 	if err != nil {
 		return nil, err

builtin/logical/database/backend_test.go

@@ -71,7 +71,7 @@ func preparePostgresTestContainer(t *testing.T, s logical.Storage, b logical.Bac
 		})
 		if err != nil || (resp != nil && resp.IsError()) {
 			// It's likely not up and running yet, so return error and try again
-			return fmt.Errorf("err:%#v resp:%#v", err, resp)
+			return fmt.Errorf("err:%#v resp:%+v", err, resp)
 		}
 		if resp == nil {
 			t.Fatal("expected warning")

builtin/logical/database/path_rotate_credentials.go

@@ -5,6 +5,10 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/hashicorp/vault/sdk/database/dbplugin"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/queue"
@@ -72,6 +76,16 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
 			return nil, err
 		}
 
+		defer func() {
+			// Close the plugin
+			db.closed = true
+			if err := db.Database.Close(); err != nil {
+				b.Logger().Error("error closing the database plugin connection", "err", err)
+			}
+			// Even on error, still remove the connection
+			delete(b.connections, name)
+		}()
+
 		// Take out the backend lock since we are swapping out the connection
 		b.Lock()
 		defer b.Unlock()
@@ -80,12 +94,44 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
 		db.Lock()
 		defer db.Unlock()
 
-		connectionDetails, err := db.RotateRootCredentials(ctx, config.RootCredentialsRotateStatements)
+		// Generate new credentials
+		userName := config.ConnectionDetails["username"].(string)
+		oldPassword := config.ConnectionDetails["password"].(string)
+		newPassword, err := db.GenerateCredentials(ctx)
+		if err != nil {
+			return nil, err
+		}
+		config.ConnectionDetails["password"] = newPassword
+
+		// Write a WAL entry
+		walID, err := framework.PutWAL(ctx, req.Storage, rotateRootWALKey, &rotateRootCredentialsWAL{
+			ConnectionName: name,
+			UserName:       userName,
+			OldPassword:    oldPassword,
+			NewPassword:    newPassword,
+		})
 		if err != nil {
 			return nil, err
 		}
 
-		config.ConnectionDetails = connectionDetails
+		// Attempt to use SetCredentials for the root credential rotation
+		statements := dbplugin.Statements{Rotation: config.RootCredentialsRotateStatements}
+		userConfig := dbplugin.StaticUserConfig{
+			Username: userName,
+			Password: newPassword,
+		}
+		if _, _, err := db.SetCredentials(ctx, statements, userConfig); err != nil {
+			if status.Code(err) == codes.Unimplemented {
+				// Fall back to using RotateRootCredentials if unimplemented
+				config.ConnectionDetails, err = db.RotateRootCredentials(ctx,
+					config.RootCredentialsRotateStatements)
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// Update storage with the new root credentials
 		entry, err := logical.StorageEntryJSON(fmt.Sprintf("config/%s", name), config)
 		if err != nil {
 			return nil, err
@@ -94,17 +140,15 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
 			return nil, err
 		}
 
-		// Close the plugin
-		db.closed = true
-		if err := db.Database.Close(); err != nil {
-			b.Logger().Error("error closing the database plugin connection", "err", err)
+		// Delete the WAL entry after successfully rotating root credentials
+		if err := framework.DeleteWAL(ctx, req.Storage, walID); err != nil {
+			b.Logger().Warn("unable to delete WAL", "error", err, "WAL ID", walID)
 		}
-		// Even on error, still remove the connection
-		delete(b.connections, name)
 
 		return nil, nil
 	}
 }
+
 func (b *databaseBackend) pathRotateRoleCredentialsUpdate() framework.OperationFunc {
 	return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
 		name := data.Get("name").(string)

builtin/logical/database/rollback.go

@@ -0,0 +1,112 @@
+package database
+
+import (
+	"context"
+	"errors"
+
+	"github.com/hashicorp/vault/sdk/database/dbplugin"
+	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/mitchellh/mapstructure"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// WAL storage key used for the rollback of root database credentials
+const rotateRootWALKey = "rotateRootWALKey"
+
+// WAL entry used for the rollback of root database credentials
+type rotateRootCredentialsWAL struct {
+	ConnectionName string
+	UserName       string
+	NewPassword    string
+	OldPassword    string
+}
+
+// walRollback handles WAL entries that result from partial failures
+// to rotate the root credentials of a database. It is responsible
+// for rolling back root database credentials when doing so would
+// reconcile the credentials with Vault storage.
+func (b *databaseBackend) walRollback(ctx context.Context, req *logical.Request, kind string, data interface{}) error {
+	if kind != rotateRootWALKey {
+		return errors.New("unknown type to rollback")
+	}
+
+	// Decode the WAL data
+	var entry rotateRootCredentialsWAL
+	if err := mapstructure.Decode(data, &entry); err != nil {
+		return err
+	}
+
+	// Get the current database configuration from storage
+	config, err := b.DatabaseConfig(ctx, req.Storage, entry.ConnectionName)
+	if err != nil {
+		return err
+	}
+
+	// The password in storage doesn't match the new password
+	// in the WAL entry. This means there was a partial failure
+	// to update either the database or storage.
+	if config.ConnectionDetails["password"] != entry.NewPassword {
+		// Clear any cached connection to inform the rollback decision
+		err := b.ClearConnection(entry.ConnectionName)
+		if err != nil {
+			return err
+		}
+
+		// Attempt to get a connection with the current configuration.
+		// If successful, the WAL entry can be deleted. This means
+		// the root credentials are the same according to the database
+		// and storage.
+		_, err = b.GetConnection(ctx, req.Storage, entry.ConnectionName)
+		if err == nil {
+			return nil
+		}
+
+		return b.rollbackDatabaseCredentials(ctx, config, entry)
+	}
+
+	// The password in storage matches the new password in
+	// the WAL entry, so there is nothing to roll back. This
+	// means the new password was successfully updated in the
+	// database and storage, but the WAL wasn't deleted.
+	return nil
+}
+
+// rollbackDatabaseCredentials rolls back root database credentials for
+// the connection associated with the passed WAL entry. It will creates
+// a connection to the database using the WAL entry new password in
+// order to alter the password to be the WAL entry old password.
+func (b *databaseBackend) rollbackDatabaseCredentials(ctx context.Context, config *DatabaseConfig, entry rotateRootCredentialsWAL) error {
+	// Attempt to get a connection with the WAL entry new password.
+	config.ConnectionDetails["password"] = entry.NewPassword
+	dbc, err := b.GetConnectionWithConfig(ctx, entry.ConnectionName, config)
+	if err != nil {
+		return err
+	}
+
+	// Ensure the connection used to roll back the database password is not cached
+	defer func() {
+		if err := b.ClearConnection(entry.ConnectionName); err != nil {
+			b.Logger().Error("error closing database plugin connection", "err", err)
+		}
+	}()
+
+	// Roll back the database password to the WAL entry old password
+	statements := dbplugin.Statements{Rotation: config.RootCredentialsRotateStatements}
+	userConfig := dbplugin.StaticUserConfig{
+		Username: entry.UserName,
+		Password: entry.OldPassword,
+	}
+	if _, _, err := dbc.SetCredentials(ctx, statements, userConfig); err != nil {
+		// If the database plugin doesn't implement SetCredentials, the root
+		// credentials can't be rolled back. This means the root credential
+		// rotation happened via the plugin RotateRootCredentials RPC.
+		if status.Code(err) == codes.Unimplemented {
+			return nil
+		}
+
+		return err
+	}
+
+	return nil
+}