Merge branch 'master' into entity-bulk-delete

.circleci/config/jobs/build-go-dev.yml

@@ -12,7 +12,7 @@ steps:
         mkdir ./pkg
 
         # Build dev binary
-        make bootstrap dev
+        make ci-bootstrap dev
   - persist_to_workspace:
       root: .
       paths:

CHANGELOG.md

@@ -3,11 +3,16 @@
 CHANGES:
 
 * token: Token renewals will now return token policies within the `token_policies` , identity policies within `identity_policies`, and the full policy set within `policies`. [[GH-8535](https://github.com/hashicorp/vault/pull/8535)]
+* kv: Return the value of delete_version_after when reading kv/config, even if it is set to the default. [[GH-42](https://github.com/hashicorp/vault-plugin-secrets-kv/pull/42)]
 
 IMPROVEMENTS:
 
 * secrets/gcp: Support BigQuery dataset ACLs in absence of IAM endpoints [[GH-78](https://github.com/hashicorp/vault-plugin-secrets-gcp/pull/78)]
 
+BUG FIXES:
+
+* secrets/database: Fix issue where rotating root database credentials while Vault's storage backend is unavailable causes Vault to lose access to the database [[GH-8782](https://github.com/hashicorp/vault/pull/8782)]
+
 ## 1.4.1 (TBD)
 
 CHANGES: 
@@ -20,10 +25,16 @@ IMPROVEMENTS:
 
 BUG FIXES:
 
-* config/seal: Fix segfault when seal block is removed[[GH-8517](https://github.com/hashicorp/vault/pull/8517)]
+* auth/okta: Fix MFA regression (introduced in [GH-8143](https://github.com/hashicorp/vault/pull/8143)) from 1.4.0 [[GH-8807](https://github.com/hashicorp/vault/pull/8807)]
+* auth/userpass: Fix upgrade value for `token_bound_cidrs` being ignored due to incorrect key provided [[GH-8826](https://github.com/hashicorp/vault/pull/8826/files)]
+* config/seal: Fix segfault when seal block is removed [[GH-8517](https://github.com/hashicorp/vault/pull/8517)]
+* core: Fix an issue where users attempting to build Vault could receive Go module checksum errors [[GH-8770](https://github.com/hashicorp/vault/pull/8770)]
 * core: Fix blocked requests if a SIGHUP is issued during a long-running request has the state lock held. 
   Also fixes deadlock that can happen if `vault debug` with the config target is ran during this time.
   [[GH-8755](https://github.com/hashicorp/vault/pull/8755)]
+* http: Fix superflous call messages from the http package on logs caused by missing returns after
+  `respondError` calls [[GH-8796](https://github.com/hashicorp/vault/pull/8796)]
+* raft: Fix panic that could occur if `disable_clustering` was set to true on Raft storage cluster [[GH-8784](https://github.com/hashicorp/vault/pull/8784)]
 * sys/wrapping: Allow unwrapping of wrapping tokens which contain nil data [[GH-8714](https://github.com/hashicorp/vault/pull/8714)]
 
 ## 1.4.0 (April 7th, 2020)
@@ -176,7 +187,7 @@ IMPROVEMENTS:
 BUG FIXES:
 
 * auth/azure: Fix Azure compute client to use correct base URL [[GH-8072](https://github.com/hashicorp/vault/pull/8072)]
-* auth/ldap: Fix renewal of tokens without cofigured policies that are
+* auth/ldap: Fix renewal of tokens without configured policies that are
   generated by an LDAP login [[GH-8072](https://github.com/hashicorp/vault/pull/8072)]
 * auth/okta: Fix renewal of tokens without configured policies that are
   generated by an Okta login [[GH-8072](https://github.com/hashicorp/vault/pull/8072)]

Makefile

@@ -9,12 +9,12 @@ TEST_TIMEOUT?=45m
 EXTENDED_TEST_TIMEOUT=60m
 INTEG_TEST_TIMEOUT=120m
 VETARGS?=-asmdecl -atomic -bool -buildtags -copylocks -methods -nilfunc -printf -rangeloops -shift -structtags -unsafeptr
-EXTERNAL_TOOLS=\
-	golang.org/x/tools/cmd/goimports \
+EXTERNAL_TOOLS_CI=\
 	github.com/elazarl/go-bindata-assetfs/... \
 	github.com/hashicorp/go-bindata/... \
-	github.com/mitchellh/gox \
-	github.com/kardianos/govendor \
+	github.com/mitchellh/gox
+EXTERNAL_TOOLS=\
+	golang.org/x/tools/cmd/goimports \
 	github.com/client9/misspell/cmd/misspell \
 	github.com/golangci/golangci-lint/cmd/golangci-lint
 GOFMT_FILES?=$$(find . -name '*.go' | grep -v pb.go | grep -v vendor)
@@ -126,8 +126,15 @@ ci-config:
 ci-verify:
 	@$(MAKE) -C .circleci ci-verify
 
-# bootstrap the build by downloading additional tools
-bootstrap:
+# bootstrap the build by downloading additional tools needed to build
+ci-bootstrap:
+	@for tool in  $(EXTERNAL_TOOLS_CI) ; do \
+		echo "Installing/Updating $$tool" ; \
+		GO111MODULE=off $(GO_CMD) get -u $$tool; \
+	done
+
+# bootstrap the build by downloading additional tools that may be used by devs
+bootstrap: ci-bootstrap
 	@for tool in  $(EXTERNAL_TOOLS) ; do \
 		echo "Installing/Updating $$tool" ; \
 		GO111MODULE=off $(GO_CMD) get -u $$tool; \
@@ -274,6 +281,6 @@ publish-commit:
 	@[ -n "$(PUBLISH_VERSION)" ] || { echo "You must set PUBLISH_VERSION to the version in semver-like format."; exit 1; }
 	set -x; $(GPG_KEY_VARS) && git commit --allow-empty --gpg-sign=$$GIT_GPG_KEY_ID -m 'release: publish v$(PUBLISH_VERSION)'
 
-.PHONY: bin default prep test vet bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin static-assets ember-dist ember-dist-dev static-dist static-dist-dev assetcheck check-vault-in-path check-browserstack-creds test-ui-browserstack stage-commit publish-commit
+.PHONY: bin default prep test vet ci-bootstrap bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin static-assets ember-dist ember-dist-dev static-dist static-dist-dev assetcheck check-vault-in-path check-browserstack-creds test-ui-browserstack stage-commit publish-commit
 
 .NOTPARALLEL: ember-dist ember-dist-dev static-assets

builtin/credential/okta/backend.go

@@ -100,7 +100,7 @@ func (b *backend) Login(ctx context.Context, req *logical.Request, username stri
 		StateToken   string         `json:"stateToken"`
 	}
 
-	authReq, err := shim.NewRequest("POST", "/api/v1/authn", map[string]interface{}{
+	authReq, err := shim.NewRequest("POST", "authn", map[string]interface{}{
 		"username": username,
 		"password": password,
 	})

builtin/credential/okta/backend_test.go

@@ -15,6 +15,15 @@ import (
 	"github.com/hashicorp/vault/sdk/logical"
 )
 
+// To run this test, set the following env variables:
+// VAULT_ACC=1
+// OKTA_ORG=dev-219337
+// OKTA_API_TOKEN=<find in 1password>
+// [email protected]
+// OKTA_PASSWORD=<find in 1password>
+//
+// You will need to install the Okta client app on your mobile device and
+// setup MFA.
 func TestBackend_Config(t *testing.T) {
 	defaultLeaseTTLVal := time.Hour * 12
 	maxLeaseTTLVal := time.Hour * 24

builtin/credential/okta/path_config.go

@@ -6,6 +6,7 @@ import (
 	"github.com/hashicorp/go-cleanhttp"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	oktaold "github.com/chrismalek/oktasdk-go/okta"
@@ -282,6 +283,9 @@ func (new *oktaShimNew) Client() *oktanew.Client {
 }
 
 func (new *oktaShimNew) NewRequest(method string, url string, body interface{}) (*http.Request, error) {
+	if !strings.HasPrefix(url, "/") {
+		url = "/api/v1/" + url
+	}
 	return new.client.GetRequestExecutor().NewRequest(method, url, body)
 }
 

builtin/credential/userpass/path_users.go

@@ -231,7 +231,7 @@ func (b *backend) userCreateUpdate(ctx context.Context, req *logical.Request, d
 			return logical.ErrorResponse(err.Error()), nil
 		}
 
-		if err := tokenutil.UpgradeValue(d, "bound_cidrs", "token_bound_cirs", &userEntry.BoundCIDRs, &userEntry.TokenBoundCIDRs); err != nil {
+		if err := tokenutil.UpgradeValue(d, "bound_cidrs", "token_bound_cidrs", &userEntry.BoundCIDRs, &userEntry.TokenBoundCIDRs); err != nil {
 			return logical.ErrorResponse(err.Error()), nil
 		}
 	}

builtin/logical/database/backend.go

@@ -6,6 +6,7 @@ import (
 	"net/rpc"
 	"strings"
 	"sync"
+	"time"
 
 	log "github.com/hashicorp/go-hclog"
 
@@ -24,6 +25,7 @@ const (
 	databaseConfigPath     = "database/config/"
 	databaseRolePath       = "role/"
 	databaseStaticRolePath = "static-role/"
+	minRootCredRollbackAge = 1 * time.Minute
 )
 
 type dbPluginInstance struct {
@@ -93,9 +95,11 @@ func Backend(conf *logical.BackendConfig) *databaseBackend {
 		Secrets: []*framework.Secret{
 			secretCreds(&b),
 		},
-		Clean:       b.clean,
-		Invalidate:  b.invalidate,
-		BackendType: logical.TypeLogical,
+		Clean:             b.clean,
+		Invalidate:        b.invalidate,
+		WALRollback:       b.walRollback,
+		WALRollbackMinAge: minRootCredRollbackAge,
+		BackendType:       logical.TypeLogical,
 	}
 
 	b.logger = conf.Logger
@@ -223,6 +227,15 @@ func (b *databaseBackend) invalidate(ctx context.Context, key string) {
 }
 
 func (b *databaseBackend) GetConnection(ctx context.Context, s logical.Storage, name string) (*dbPluginInstance, error) {
+	config, err := b.DatabaseConfig(ctx, s, name)
+	if err != nil {
+		return nil, err
+	}
+
+	return b.GetConnectionWithConfig(ctx, name, config)
+}
+
+func (b *databaseBackend) GetConnectionWithConfig(ctx context.Context, name string, config *DatabaseConfig) (*dbPluginInstance, error) {
 	b.RLock()
 	unlockFunc := b.RUnlock
 	defer func() { unlockFunc() }()
@@ -242,11 +255,6 @@ func (b *databaseBackend) GetConnection(ctx context.Context, s logical.Storage,
 		return db, nil
 	}
 
-	config, err := b.DatabaseConfig(ctx, s, name)
-	if err != nil {
-		return nil, err
-	}
-
 	dbp, err := dbplugin.PluginFactory(ctx, config.PluginName, b.System(), b.logger)
 	if err != nil {
 		return nil, err

builtin/logical/database/backend_test.go

@@ -71,7 +71,7 @@ func preparePostgresTestContainer(t *testing.T, s logical.Storage, b logical.Bac
 		})
 		if err != nil || (resp != nil && resp.IsError()) {
 			// It's likely not up and running yet, so return error and try again
-			return fmt.Errorf("err:%#v resp:%#v", err, resp)
+			return fmt.Errorf("err:%#v resp:%+v", err, resp)
 		}
 		if resp == nil {
 			t.Fatal("expected warning")

builtin/logical/database/path_rotate_credentials.go

@@ -5,6 +5,10 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/hashicorp/vault/sdk/database/dbplugin"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/queue"
@@ -72,6 +76,16 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
 			return nil, err
 		}
 
+		defer func() {
+			// Close the plugin
+			db.closed = true
+			if err := db.Database.Close(); err != nil {
+				b.Logger().Error("error closing the database plugin connection", "err", err)
+			}
+			// Even on error, still remove the connection
+			delete(b.connections, name)
+		}()
+
 		// Take out the backend lock since we are swapping out the connection
 		b.Lock()
 		defer b.Unlock()
@@ -80,12 +94,44 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
 		db.Lock()
 		defer db.Unlock()
 
-		connectionDetails, err := db.RotateRootCredentials(ctx, config.RootCredentialsRotateStatements)
+		// Generate new credentials
+		userName := config.ConnectionDetails["username"].(string)
+		oldPassword := config.ConnectionDetails["password"].(string)
+		newPassword, err := db.GenerateCredentials(ctx)
+		if err != nil {
+			return nil, err
+		}
+		config.ConnectionDetails["password"] = newPassword
+
+		// Write a WAL entry
+		walID, err := framework.PutWAL(ctx, req.Storage, rotateRootWALKey, &rotateRootCredentialsWAL{
+			ConnectionName: name,
+			UserName:       userName,
+			OldPassword:    oldPassword,
+			NewPassword:    newPassword,
+		})
 		if err != nil {
 			return nil, err
 		}
 
-		config.ConnectionDetails = connectionDetails
+		// Attempt to use SetCredentials for the root credential rotation
+		statements := dbplugin.Statements{Rotation: config.RootCredentialsRotateStatements}
+		userConfig := dbplugin.StaticUserConfig{
+			Username: userName,
+			Password: newPassword,
+		}
+		if _, _, err := db.SetCredentials(ctx, statements, userConfig); err != nil {
+			if status.Code(err) == codes.Unimplemented {
+				// Fall back to using RotateRootCredentials if unimplemented
+				config.ConnectionDetails, err = db.RotateRootCredentials(ctx,
+					config.RootCredentialsRotateStatements)
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// Update storage with the new root credentials
 		entry, err := logical.StorageEntryJSON(fmt.Sprintf("config/%s", name), config)
 		if err != nil {
 			return nil, err
@@ -94,17 +140,15 @@ func (b *databaseBackend) pathRotateCredentialsUpdate() framework.OperationFunc
 			return nil, err
 		}
 
-		// Close the plugin
-		db.closed = true
-		if err := db.Database.Close(); err != nil {
-			b.Logger().Error("error closing the database plugin connection", "err", err)
+		// Delete the WAL entry after successfully rotating root credentials
+		if err := framework.DeleteWAL(ctx, req.Storage, walID); err != nil {
+			b.Logger().Warn("unable to delete WAL", "error", err, "WAL ID", walID)
 		}
-		// Even on error, still remove the connection
-		delete(b.connections, name)
 
 		return nil, nil
 	}
 }
+
 func (b *databaseBackend) pathRotateRoleCredentialsUpdate() framework.OperationFunc {
 	return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
 		name := data.Get("name").(string)