Merge remote-tracking branch 'origin/master' into sys-config-state

CHANGELOG.md

@@ -2,12 +2,17 @@
 
 FEATURES:
 
- * **Stackdriver Metrics Sync**: Vault can now send metrics to
+ * **Stackdriver Metrics Sink**: Vault can now send metrics to
    [Stackdriver](https://cloud.google.com/stackdriver/). See the [configuration
    documentation](https://www.vaultproject.io/docs/config/index.html) for
    details. [GH-6957]
-
+ * Transit: Signing and verification is now supported with the P-384
+   (secp384r1) and P-521 (secp521r1) ECDSA curves [GH-7551]
+ * Transit: Encryption and decryption is now supported via AES128-GCM96
+   [GH-7555]
+
 CHANGES: 
+
  * sys/seal-status now has a `storage_type` field denoting what type of storage
    the cluster is configured to use
 
@@ -19,6 +24,29 @@ IMPROVEMENTS:
  * secrets/aws: The root config can now be read [GH-7245]
  * storage/cassandra: Improve storage efficiency by eliminating unnecessary
    copies of value data [GH-7199]
+ * sys: Add a new `sys/host-info` endpoint for querying information about 
+   the host [GH-7330]
+ * sys: Add a new set of endpoints under `sys/pprof/` that allows profiling
+   information to be extracted [GH-7473]
+ * replication (enterprise): Write-Ahead-Log entries will not duplicate the
+   data belonging to the encompassing physical entries of the transaction,
+   thereby improving the performance and storage capacity.
+
+BUG FIXES:
+
+ * agent: Fix handling of gzipped responses [GH-7470]
+ * auth/gcp: Fix a bug where region information in instance groups names could
+   cause an authorization attempt to fail [GCP-74]
+ * cli: Fix a bug where a token of an unknown format (e.g. in ~/.vault-token)
+   could cause confusing error messages during `vault login` [GH-7508]
+ * identity: Add required field `response_types_supported` to identity token
+   `.well-known/openid-configuration` response [GH-7533]
+ * identity (enterprise): Fixed identity case sensitive loading in secondary
+   cluster [GH-7327]
+ * ui: using the `wrapped_token` query param will work with `redirect_to` and
+   will automatically log in as intended [GH-7398]
+ * secret/database: Fix bug in combined DB secrets engine that can result in
+   writes to static-roles endpoints timing out [GH-7518]
 
 ## 1.2.3 (September 12, 2019)
 

README.md

@@ -59,8 +59,8 @@ Developing Vault
 --------------------
 
 If you wish to work on Vault itself or any of its built-in systems, you'll
-first need [Go](https://www.golang.org) installed on your machine (version
-1.12.1+ is *required*).
+first need [Go](https://www.golang.org) installed on your machine. Go version
+1.12.7+ is *required*. Note: version 1.13.x is not yet supported.
 
 For local dev first make sure Go is properly installed, including setting up a
 [GOPATH](https://golang.org/doc/code.html#GOPATH). Ensure that `$GOPATH/bin` is in

api/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0
 	github.com/hashicorp/vault/sdk v0.1.14-0.20190919081434-645ac174deeb
 	github.com/mitchellh/mapstructure v1.1.2
+	github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 // indirect
 	golang.org/x/net v0.0.0-20190620200207-3b0461eec859
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	gopkg.in/square/go-jose.v2 v2.3.1

api/go.sum

@@ -1,5 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -10,6 +11,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -67,6 +69,9 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 h1:wsZAb4P8F7uQSwsnxE1gk9AHCcc5U0wvyDzcLwFY0Eo=
+github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
@@ -92,8 +97,10 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU=

builtin/credential/aws/backend_e2e_test.go

@@ -38,7 +38,6 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	// Make sure that the upgrade happened, by fishing the 'config/version'
 	// entry out of storage.  We can't use core.Client.Logical().Read() to do
 	// this, because 'config/version' hasn't been exposed as a path.
-	// TODO: should we expose 'config/version' as a path?
 	version, err := core.UnderlyingStorage.Get(ctx, awsPath+"config/version")
 	if err != nil {
 		t.Fatal(err)

builtin/logical/database/rotation.go

@@ -127,112 +127,115 @@ type setCredentialsWAL struct {
 // This method loops through the priority queue, popping the highest priority
 // item until it encounters the first item that does not yet need rotation,
 // based on the current time.
-func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Storage) error {
-	for {
-		// Quit rotating credentials if shutdown has started
-		select {
-		case <-ctx.Done():
-			return nil
-		default:
-		}
-		item, err := b.popFromRotationQueue()
-		if err != nil {
-			if err == queue.ErrEmpty {
-				return nil
-			}
-			return err
-		}
+func (b *databaseBackend) rotateCredentials(ctx context.Context, s logical.Storage) {
+	for b.rotateCredential(ctx, s) {
+	}
+}
 
-		// Guard against possible nil item
-		if item == nil {
-			return nil
+func (b *databaseBackend) rotateCredential(ctx context.Context, s logical.Storage) bool {
+	// Quit rotating credentials if shutdown has started
+	select {
+	case <-ctx.Done():
+		return false
+	default:
+	}
+	item, err := b.popFromRotationQueue()
+	if err != nil {
+		if err != queue.ErrEmpty {
+			b.logger.Error("error popping item from queue", "err", err)
 		}
+		return false
+	}
 
-		// Grab the exclusive lock for this Role, to make sure we don't incur and
-		// writes during the rotation process
-		lock := locksutil.LockForKey(b.roleLocks, item.Key)
-		lock.Lock()
-		defer lock.Unlock()
+	// Guard against possible nil item
+	if item == nil {
+		return false
+	}
 
-		// Validate the role still exists
-		role, err := b.StaticRole(ctx, s, item.Key)
-		if err != nil {
-			b.logger.Error("unable to load role", "role", item.Key, "error", err)
-			item.Priority = time.Now().Add(10 * time.Second).Unix()
-			if err := b.pushItem(item); err != nil {
-				b.logger.Error("unable to push item on to queue", "error", err)
-			}
-			continue
-		}
-		if role == nil {
-			b.logger.Warn("role not found", "role", item.Key, "error", err)
-			continue
-		}
+	// Grab the exclusive lock for this Role, to make sure we don't incur and
+	// writes during the rotation process
+	lock := locksutil.LockForKey(b.roleLocks, item.Key)
+	lock.Lock()
+	defer lock.Unlock()
 
-		// If "now" is less than the Item priority, then this item does not need to
-		// be rotated
-		if time.Now().Unix() < item.Priority {
-			if err := b.pushItem(item); err != nil {
-				b.logger.Error("unable to push item on to queue", "error", err)
-			}
-			// Break out of the for loop
-			break
+	// Validate the role still exists
+	role, err := b.StaticRole(ctx, s, item.Key)
+	if err != nil {
+		b.logger.Error("unable to load role", "role", item.Key, "error", err)
+		item.Priority = time.Now().Add(10 * time.Second).Unix()
+		if err := b.pushItem(item); err != nil {
+			b.logger.Error("unable to push item on to queue", "error", err)
 		}
+		return true
+	}
+	if role == nil {
+		b.logger.Warn("role not found", "role", item.Key, "error", err)
+		return true
+	}
 
-		input := &setStaticAccountInput{
-			RoleName: item.Key,
-			Role:     role,
+	// If "now" is less than the Item priority, then this item does not need to
+	// be rotated
+	if time.Now().Unix() < item.Priority {
+		if err := b.pushItem(item); err != nil {
+			b.logger.Error("unable to push item on to queue", "error", err)
 		}
+		// Break out of the for loop
+		return false
+	}
 
-		// If there is a WAL entry related to this Role, the corresponding WAL ID
-		// should be stored in the Item's Value field.
-		if walID, ok := item.Value.(string); ok {
-			walEntry, err := b.findStaticWAL(ctx, s, walID)
-			if err != nil {
-				b.logger.Error("error finding static WAL", "error", err)
-				item.Priority = time.Now().Add(10 * time.Second).Unix()
-				if err := b.pushItem(item); err != nil {
-					b.logger.Error("unable to push item on to queue", "error", err)
-				}
-			}
-			if walEntry != nil && walEntry.NewPassword != "" {
-				input.Password = walEntry.NewPassword
-				input.WALID = walID
-			}
-		}
+	input := &setStaticAccountInput{
+		RoleName: item.Key,
+		Role:     role,
+	}
 
-		resp, err := b.setStaticAccount(ctx, s, input)
+	// If there is a WAL entry related to this Role, the corresponding WAL ID
+	// should be stored in the Item's Value field.
+	if walID, ok := item.Value.(string); ok {
+		walEntry, err := b.findStaticWAL(ctx, s, walID)
 		if err != nil {
-			b.logger.Error("unable to rotate credentials in periodic function", "error", err)
-			// Increment the priority enough so that the next call to this method
-			// likely will not attempt to rotate it, as a back-off of sorts
+			b.logger.Error("error finding static WAL", "error", err)
 			item.Priority = time.Now().Add(10 * time.Second).Unix()
-
-			// Preserve the WALID if it was returned
-			if resp != nil && resp.WALID != "" {
-				item.Value = resp.WALID
-			}
-
 			if err := b.pushItem(item); err != nil {
 				b.logger.Error("unable to push item on to queue", "error", err)
 			}
-			// Go to next item
-			continue
 		}
+		if walEntry != nil && walEntry.NewPassword != "" {
+			input.Password = walEntry.NewPassword
+			input.WALID = walID
+		}
+	}
 
-		lvr := resp.RotationTime
-		if lvr.IsZero() {
-			lvr = time.Now()
+	resp, err := b.setStaticAccount(ctx, s, input)
+	if err != nil {
+		b.logger.Error("unable to rotate credentials in periodic function", "error", err)
+		// Increment the priority enough so that the next call to this method
+		// likely will not attempt to rotate it, as a back-off of sorts
+		item.Priority = time.Now().Add(10 * time.Second).Unix()
+
+		// Preserve the WALID if it was returned
+		if resp != nil && resp.WALID != "" {
+			item.Value = resp.WALID
 		}
 
-		// Update priority and push updated Item to the queue
-		nextRotation := lvr.Add(role.StaticAccount.RotationPeriod)
-		item.Priority = nextRotation.Unix()
 		if err := b.pushItem(item); err != nil {
-			b.logger.Warn("unable to push item on to queue", "error", err)
+			b.logger.Error("unable to push item on to queue", "error", err)
 		}
+		// Go to next item
+		return true
 	}
-	return nil
+
+	lvr := resp.RotationTime
+	if lvr.IsZero() {
+		lvr = time.Now()
+	}
+
+	// Update priority and push updated Item to the queue
+	nextRotation := lvr.Add(role.StaticAccount.RotationPeriod)
+	item.Priority = nextRotation.Unix()
+	if err := b.pushItem(item); err != nil {
+		b.logger.Warn("unable to push item on to queue", "error", err)
+	}
+	return true
 }
 
 // findStaticWAL loads a WAL entry by ID. If found, only return the WAL if it