Merge branch 'master' into catsby-patch-3

* master:
  Add a section to the MySQL secrets plugin docs about x509 (#9757)
  Update documentation for MySQL Secrets Engine (#9671)
  Conditionally overwrite TLS parameters for MySQL secrets engine (#9729)
  Correctly mark Cassandra as not supporting static roles (#9750)
  changelog++
  pki: Allow to use not only one variable during templating in allowed_domains #8509 (#9498)
  agent/templates: update consul-template to v0.25.1 (#9626)
  Restoring the example policies for blocking sha1 (#9677)
  changelog++
  changelog++
  Document the new SSH signing algorithm option. (#9197)
  CHANGELOG-+
  CHANGELOG++
  Trail of bits 018 (#9674)

CHANGELOG.md

@@ -31,6 +31,7 @@ BUG FIXES:
 * secrets/gcp: Ensure that the IAM policy version is appropriately set after a roleset's bindings have changed. [[GH-9603](https://github.com/hashicorp/vault/pull/9603)]
 * replication (enterprise): Fix status API output incorrectly stating replication is in `idle` state.
 * core: Fix panic when printing over-long info fields at startup [[GH-9681](https://github.com/hashicorp/vault/pull/9681)]
+* core: Seal migration using the new minimal-downtime strategy didn't work properly with performance standbys. [[GH-9690](https://github.com/hashicorp/vault/pull/9690)]
 
 ## 1.5.0
 ### July 21st, 2020
@@ -122,10 +123,12 @@ BUG FIXES:
 
 BUG FIXES:
 
+* auth/okta: fix bug introduced in 1.4.0 that broke handling of external groups with > 200 members [[GH-9580](https://github.com/hashicorp/vault/pull/9580)]
 * seal/awskms: fix AWS KMS auto-unseal when AWS_ROLE_SESSION_NAME not set [[GH-9416](https://github.com/hashicorp/vault/pull/9416)]
 * secrets/aws: Fix possible issue creating access keys when using Performance Standbys  [[GH-9606](https://github.com/hashicorp/vault/pull/9606)]
 
 IMPROVEMENTS:
+* auth/aws: Retry on transient failures during AWS IAM auth login attempts [[GH-8727](https://github.com/hashicorp/vault/pull/8727)]
 * ui: Add transit key algorithms aes128-gcm96, ecdsa-p384, ecdsa-p521 to the UI. [[GH-9070](https://github.com/hashicorp/vault/pull/9070)] & [[GH-9520](https://github.com/hashicorp/vault/pull/9520)]
 
 ## 1.4.3
@@ -137,6 +140,7 @@ IMPROVEMENTS:
 * auth/kerberos: Support identities without userPrincipalName [[GH-44](https://github.com/hashicorp/vault-plugin-auth-kerberos/issues/44)]
 * core: Add the Go version used to build a Vault binary to the server message output. [[GH-9078](https://github.com/hashicorp/vault/pull/9078)]
 * secrets/database: Add static role rotation for MongoDB Atlas database plugin [[GH-9311](https://github.com/hashicorp/vault/pull/9311)]
+* physical/mysql: Require TLS or plaintext flagging in MySQL configuration [[GH-9012](https://github.com/hashicorp/vault/pull/9012)]
 * ui: Link to the Vault Changelog in the UI footer [[GH-9216](https://github.com/hashicorp/vault/pull/9216)]
 
 BUG FIXES:

builtin/logical/pki/backend_test.go

@@ -2797,7 +2797,8 @@ func TestBackend_AllowedDomainsTemplate(t *testing.T) {
 
 	// Write role PKI.
 	_, err = client.Logical().Write("pki/roles/test", map[string]interface{}{
-		"allowed_domains":          []string{"foobar.com", "zipzap.com", "{{identity.entity.aliases." + userpassAccessor + ".name}}"},
+		"allowed_domains":          []string{"foobar.com", "zipzap.com", "{{identity.entity.aliases." + userpassAccessor + ".name}}",
+										     "foo.{{identity.entity.aliases." + userpassAccessor + ".name}}.example.com"},
 		"allowed_domains_template": true,
 		"allow_bare_domains":       true,
 	})
@@ -2824,6 +2825,12 @@ func TestBackend_AllowedDomainsTemplate(t *testing.T) {
 		t.Fatal("expected error")
 	}
 
+	// Issue certificate for foo.userpassname.domain.
+	_, err = client.Logical().Write("pki/issue/test", map[string]interface{}{"common_name": "foo.userpassname.example.com"})
+	if err != nil {
+		t.Fatal("expected error")
+	}
+
 	// Set allowed_domains_template to false.
 	_, err = client.Logical().Write("pki/roles/test", map[string]interface{}{
 		"allowed_domains_template": false,

builtin/logical/pki/cert_util.go

@@ -315,8 +315,8 @@ func validateNames(b *backend, data *inputBundle, names []string) string {
 				}
 
 				if data.role.AllowedDomainsTemplate {
-					matched, _ := regexp.MatchString(`^{{.+?}}$`, currDomain)
-					if matched && data.req.EntityID != "" {
+					isTemplate, _ := framework.ValidateIdentityTemplate(currDomain)
+					if isTemplate && data.req.EntityID != "" {
 						tmpCurrDomain, err := framework.PopulateIdentityTemplate(currDomain, data.req.EntityID, b.System())
 						if err != nil {
 							continue

command/agent.go

@@ -349,6 +349,7 @@ func (c *AgentCommand) Run(args []string) int {
 					Client:  client,
 					WrapTTL: sc.WrapTTL,
 					DHType:  sc.DHType,
+					DeriveKey: sc.DeriveKey,
 					DHPath:  sc.DHPath,
 					AAD:     sc.AAD,
 				}

command/agent/cert_with_name_end_to_end_test.go

@@ -11,16 +11,16 @@ import (
 	hclog "github.com/hashicorp/go-hclog"
 
 	"github.com/hashicorp/vault/api"
+	"github.com/hashicorp/vault/sdk/helper/jsonutil"
+	"github.com/hashicorp/vault/sdk/helper/logging"
+	"github.com/hashicorp/vault/sdk/logical"
 	vaultcert "github.com/hashicorp/vault/builtin/credential/cert"
 	"github.com/hashicorp/vault/command/agent/auth"
 	agentcert "github.com/hashicorp/vault/command/agent/auth/cert"
 	"github.com/hashicorp/vault/command/agent/sink"
 	"github.com/hashicorp/vault/command/agent/sink/file"
 	"github.com/hashicorp/vault/helper/dhutil"
 	vaulthttp "github.com/hashicorp/vault/http"
-	"github.com/hashicorp/vault/sdk/helper/jsonutil"
-	"github.com/hashicorp/vault/sdk/helper/logging"
-	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/vault"
 )
 
@@ -137,6 +137,7 @@ func testCertWithNameEndToEnd(t *testing.T, ahWrapping bool) {
 		AAD:    "foobar",
 		DHType: "curve25519",
 		DHPath: dhpath,
+		DeriveKey: true,
 		Config: map[string]interface{}{
 			"path": out,
 		},
@@ -186,14 +187,17 @@ func testCertWithNameEndToEnd(t *testing.T, ahWrapping bool) {
 					continue
 				}
 
-				aesKey, err := dhutil.GenerateSharedKey(pri, resp.Curve25519PublicKey)
+				shared, err := dhutil.GenerateSharedSecret(pri, resp.Curve25519PublicKey)
+				if err != nil {
+					t.Fatal(err)
+				}
+				aesKey, err := dhutil.DeriveSharedKey(shared, pub, resp.Curve25519PublicKey)
 				if err != nil {
 					t.Fatal(err)
 				}
 				if len(aesKey) == 0 {
 					t.Fatal("got empty aes key")
 				}
-
 				val, err = dhutil.DecryptAES(aesKey, resp.EncryptedPayload, resp.Nonce, []byte("foobar"))
 				if err != nil {
 					t.Fatalf("error: %v\nresp: %v", err, string(val))

command/agent/cert_with_no_name_end_to_end_test.go

@@ -183,7 +183,7 @@ func testCertWithNoNAmeEndToEnd(t *testing.T, ahWrapping bool) {
 					continue
 				}
 
-				aesKey, err := dhutil.GenerateSharedKey(pri, resp.Curve25519PublicKey)
+				aesKey, err := dhutil.GenerateSharedSecret(pri, resp.Curve25519PublicKey)
 				if err != nil {
 					t.Fatal(err)
 				}

command/agent/config/config.go

@@ -75,6 +75,7 @@ type Sink struct {
 	WrapTTLRaw interface{}   `hcl:"wrap_ttl"`
 	WrapTTL    time.Duration `hcl:"-"`
 	DHType     string        `hcl:"dh_type"`
+	DeriveKey  bool          `hcl:"derive_key"`
 	DHPath     string        `hcl:"dh_path"`
 	AAD        string        `hcl:"aad"`
 	AADEnvVar  string        `hcl:"aad_env_var"`
@@ -395,6 +396,9 @@ func parseSinks(result *Config, list *ast.ObjectList) error {
 			if s.AAD != "" {
 				return multierror.Prefix(errors.New("specifying AAD data without 'dh_type' does not make sense"), fmt.Sprintf("sink.%s", s.Type))
 			}
+			if s.DeriveKey {
+				return multierror.Prefix(errors.New("specifying 'derive_key' data without 'dh_type' does not make sense"), fmt.Sprintf("sink.%s", s.Type))
+			}
 		case s.DHPath != "" && s.DHType != "":
 		default:
 			return multierror.Prefix(errors.New("'dh_type' and 'dh_path' must be specified together"), fmt.Sprintf("sink.%s", s.Type))

command/agent/config/config_test.go

@@ -143,6 +143,7 @@ func TestLoadConfigFile(t *testing.T) {
 					DHType:  "curve25519",
 					DHPath:  "/tmp/file-foo-dhpath2",
 					AAD:     "aad",
+					DeriveKey: true,
 					Config: map[string]interface{}{
 						"path": "/tmp/file-bar",
 					},

command/agent/config/test-fixtures/config-embedded-type.hcl

@@ -23,6 +23,7 @@ auto_auth {
 		aad_env_var = "TEST_AAD_ENV"
 		dh_type = "curve25519"
 		dh_path = "/tmp/file-foo-dhpath2"
+		derive_key = true
 		config = {
 			path = "/tmp/file-bar"
 		}

command/agent/config/test-fixtures/config.hcl

@@ -25,6 +25,7 @@ auto_auth {
 		aad_env_var = "TEST_AAD_ENV"
 		dh_type = "curve25519"
 		dh_path = "/tmp/file-foo-dhpath2"
+		derive_key = true
 		config = {
 			path = "/tmp/file-bar"
 		}

command/agent/jwt_end_to_end_test.go

@@ -158,6 +158,7 @@ func testJWTEndToEnd(t *testing.T, ahWrapping bool) {
 		AAD:    "foobar",
 		DHType: "curve25519",
 		DHPath: dhpath,
+		DeriveKey: true,
 		Config: map[string]interface{}{
 			"path": out,
 		},
@@ -231,7 +232,11 @@ func testJWTEndToEnd(t *testing.T, ahWrapping bool) {
 					continue
 				}
 
-				aesKey, err := dhutil.GenerateSharedKey(pri, resp.Curve25519PublicKey)
+				shared, err := dhutil.GenerateSharedSecret(pri, resp.Curve25519PublicKey)
+				if err != nil {
+					t.Fatal(err)
+				}
+				aesKey, err := dhutil.DeriveSharedKey(shared, pub, resp.Curve25519PublicKey)
 				if err != nil {
 					t.Fatal(err)
 				}

command/agent/sink/sink.go

@@ -32,6 +32,7 @@ type SinkConfig struct {
 	WrapTTL            time.Duration
 	DHType             string
 	DHPath             string
+	DeriveKey          bool
 	AAD                string
 	cachedRemotePubKey []byte
 	cachedPubKey       []byte
@@ -205,7 +206,16 @@ func (s *SinkConfig) encryptToken(token string) (string, error) {
 		resp.Curve25519PublicKey = s.cachedPubKey
 	}
 
-	aesKey, err = dhutil.GenerateSharedKey(s.cachedPriKey, s.cachedRemotePubKey)
+	secret, err := dhutil.GenerateSharedSecret(s.cachedPriKey, s.cachedRemotePubKey)
+	if err != nil {
+		return "", errwrap.Wrapf("error calculating shared key: {{err}}", err)
+	}
+	if s.DeriveKey {
+		aesKey, err = dhutil.DeriveSharedKey(secret, s.cachedPubKey, s.cachedRemotePubKey)
+	} else {
+		aesKey = secret
+	}
+
 	if err != nil {
 		return "", errwrap.Wrapf("error deriving shared key: {{err}}", err)
 	}

go.mod

@@ -48,7 +48,7 @@ require (
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-metrics-stackdriver v0.2.0
-	github.com/hashicorp/consul-template v0.25.0
+	github.com/hashicorp/consul-template v0.25.1
 	github.com/hashicorp/consul/api v1.4.0
 	github.com/hashicorp/errwrap v1.0.0
 	github.com/hashicorp/go-bindata v3.0.8-0.20180209072458-bf7910af8997+incompatible

go.sum

@@ -431,8 +431,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
-github.com/hashicorp/consul-template v0.25.0 h1:wsnv4jSqBIVzlg6U0wNg+ePzfrsF3Vi9MqIqDEUrg9U=
-github.com/hashicorp/consul-template v0.25.0/go.mod h1:/vUsrJvDuuQHcxEw0zik+YXTS7ZKWZjQeaQhshBmfH0=
+github.com/hashicorp/consul-template v0.25.1 h1:+D2s8eyRqWyX7GPNxeUi8tsyh8pRn3J6k8giEchPfKQ=
+github.com/hashicorp/consul-template v0.25.1/go.mod h1:/vUsrJvDuuQHcxEw0zik+YXTS7ZKWZjQeaQhshBmfH0=
 github.com/hashicorp/consul/api v1.4.0 h1:jfESivXnO5uLdH650JU/6AnjRoHrLhULq0FnC3Kp9EY=
 github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=

helper/dhutil/dhutil.go

@@ -1,11 +1,15 @@
 package dhutil
 
 import (
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"golang.org/x/crypto/hkdf"
 	"io"
 
 	"golang.org/x/crypto/curve25519"
@@ -34,23 +38,60 @@ func GeneratePublicPrivateKey() ([]byte, []byte, error) {
 	return public[:], scalar[:], nil
 }
 
-// generateSharedKey uses the private key and the other party's public key to
+// GenerateSharedSecret uses the private key and the other party's public key to
 // generate the shared secret.
-func GenerateSharedKey(ourPrivate, theirPublic []byte) ([]byte, error) {
+func GenerateSharedSecret(ourPrivate, theirPublic []byte) ([]byte, error) {
 	if len(ourPrivate) != 32 {
 		return nil, fmt.Errorf("invalid private key length: %d", len(ourPrivate))
 	}
 	if len(theirPublic) != 32 {
 		return nil, fmt.Errorf("invalid public key length: %d", len(theirPublic))
 	}
 
-	var scalar, pub, secret [32]byte
-	copy(scalar[:], ourPrivate)
-	copy(pub[:], theirPublic)
+	return curve25519.X25519(ourPrivate, theirPublic)
+}
+
+// DeriveSharedKey uses HKDF to derive a key from a shared secret and public keys
+func DeriveSharedKey(secret, ourPublic, theirPublic []byte) ([]byte, error) {
+	// Derive the final key from the HKDF of the secret and public keys.
+
+/*
+	Internally, HKDF hashes the secret and two public keys. If Alice and Bob are doing DH key exchange, Alice calculates:
+
+	HKDF(secret, A, B) since ourPublic is A.
+
+		Bob calculates HKDF(secret, B, A), since Bob's ours is B. That produces a different value. Now we only care
+	    that both public keys participate in the derivation, so simply sorting them so they are in a consistent
+	    numerical order (either one would do) arrives at an agreed value.
+*/
+
+	var pub1 []byte
+	var pub2 []byte
+	switch bytes.Compare(ourPublic, theirPublic) {
+	case 0:
+		return nil, errors.New("same public key supplied for both participants")
+	case -1:
+		pub1 = ourPublic
+		pub2 = theirPublic
+	case 1:
+		pub1 = theirPublic
+		pub2 = ourPublic
+	}
+
+	kio := hkdf.New(sha256.New, secret, pub1, pub2)
 
-	curve25519.ScalarMult(&secret, &scalar, &pub)
+	var key [32]byte
+	n, err := io.ReadFull(kio, key[:])
+	if err != nil {
+		// Don't return the key along with the error to prevent misuse
+		return nil, err
+	}
+	if n != 32 {
+		return nil, errors.New("short read from hkdf")
+	}
+	fmt.Printf("Key: %s\n", hex.EncodeToString(key[:]))
 
-	return secret[:], nil
+	return key[:], nil
 }
 
 // Use AES256-GCM to encrypt some plaintext with a provided key. The returned values are

helper/dhutil/dhutil_test.go

@@ -0,0 +1 @@
+package dhutil

plugins/database/mysql/connection_producer.go

@@ -218,7 +218,9 @@ func (c *mySQLConnectionProducer) addTLStoDSN() (connURL string, err error) {
 		return "", fmt.Errorf("unable to parse connectionURL: %s", err)
 	}
 
-	config.TLSConfig = c.tlsConfigName
+	if len(c.tlsConfigName) > 0 {
+		config.TLSConfig = c.tlsConfigName
+	}
 
 	connURL = config.FormatDSN()
 

plugins/database/mysql/connection_producer_test.go

@@ -45,6 +45,11 @@ func Test_addTLStoDSN(t *testing.T) {
 			tlsConfigName:  "tlsTest101",
 			expectedResult: "user:pa?ssword?@tcp(localhost:3306)/test?tls=tlsTest101&foo=bar",
 		},
+		"tls, valid tls parameter in query string": {
+			rootUrl:        "user:password@tcp(localhost:3306)/test?tls=true",
+			tlsConfigName:  "",
+			expectedResult: "user:password@tcp(localhost:3306)/test?tls=true",
+		},
 	}
 
 	for name, test := range tests {