Merge remote-tracking branch 'oss/master' into db-cred-rotate

* oss/master: (35 commits)
  helper/gpgkeys: fix for vault 1.10 (#4038)
  Move local cluster parameters to atomic values to fix some potential data races (#4036)
  Port some replicated cluster changes from ent (#4037)
  Add core object to policy store for some ent uses
  changelog++
  Configurable Consul Service Address (#3971)
  Fix certutil test
  Fixed a broken link (#4032)
  Update comment to replication consts
  Add a helpful comment to replication consts
  changelog++
  auth/aws: Add functional test for detached RSA signature (#4031)
  Change Go min version check
  changelog++
  Revert Go dep to 1.9
  *Partially* revert "Remove now-unneeded PKCS8 code and update certutil tests for Go 1.10"
  Revert "Remove unneeded looping since Go 1.10 cover it already (#4010)"
  Bump pkcs7 library version to fix #4024
  Revert "Switch to a forked copy of pkcs7 to fix aws pkcs7 verification error (#4024)"
  changelog++
  ...

.travis.yml

@@ -7,7 +7,7 @@ services:
   - docker
 
 go:
-  - "1.10"
+  - "1.9"
 
 matrix:
   allow_failures:

CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.9.5 (Soon)
+
+IMPROVEMENTS:
+
+ * auth: Allow sending default_lease_ttl and max_lease_ttl values when enabling
+   auth methods. [GH-4019]
+ * secret/database: Add list functionality to `database/config` endpoint
+   [GH-4026]
+ * physical/consul: Allow setting a specific service address [GH-3971]
+
+BUG FIXES:
+
+ * auth/aws: Update libraries to fix regression verifying PKCS#7 identity
+   documents [GH-4014]
+ * listener: Revert to Go 1.9 for now to allow certificates with non-DNS names
+   in their DNS SANs to be used for Vault's TLS connections [GH-4028]
+ * replication: Fix issue with a performance secondary/DR primary node losing
+   its DR primary status when performing an update-primary operation
+ * replication: Fix issue where performance secondaries could be unable to
+   automatically connect to a performance primary after that performance
+   primary has been promoted to a DR primary from a DR secondary
+
 ## 0.9.4 (February 20th, 2018)
 
 SECURITY:
@@ -63,6 +85,10 @@ BUG FIXES:
  * auth/token: Token creation via the CLI no longer forces periodic token
    creation. Passing an explicit zero value for the period no longer create
    periodic tokens. [GH-3880]
+ * command: Fix interpreted formatting directives when printing raw fields
+   [GH-4005]
+ * command: Correctly format output when using -field and -format flags at the
+   same time [GH-3987]
  * command/rekey: Re-add lost `stored-shares` parameter [GH-3974]
  * command/ssh: Create and reuse the api client [GH-3909]
  * command/status: Fix panic when status returns 500 from leadership lookup

Makefile

@@ -11,7 +11,7 @@ EXTERNAL_TOOLS=\
 BUILD_TAGS?=vault
 GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
 
-GO_VERSION_MIN=1.10
+GO_VERSION_MIN=1.9
 
 default: dev
 

README.md

@@ -59,7 +59,7 @@ Developing Vault
 --------------------
 
 If you wish to work on Vault itself or any of its built-in systems, you'll
-first need [Go](https://www.golang.org) installed on your machine (version 1.10+
+first need [Go](https://www.golang.org) installed on your machine (version 1.9+
 is *required*).
 
 For local dev first make sure Go is properly installed, including setting up a

api/sys_auth.go

@@ -91,7 +91,9 @@ type EnableAuthOptions struct {
 }
 
 type AuthConfigInput struct {
-	PluginName string `json:"plugin_name,omitempty" structs:"plugin_name,omitempty" mapstructure:"plugin_name"`
+	DefaultLeaseTTL string `json:"default_lease_ttl" structs:"default_lease_ttl" mapstructure:"default_lease_ttl"`
+	MaxLeaseTTL     string `json:"max_lease_ttl" structs:"max_lease_ttl" mapstructure:"max_lease_ttl"`
+	PluginName      string `json:"plugin_name,omitempty" structs:"plugin_name,omitempty" mapstructure:"plugin_name"`
 }
 
 type AuthMount struct {

builtin/credential/aws/backend_test.go

@@ -958,18 +958,30 @@ func TestBackend_PathBlacklistRoleTag(t *testing.T) {
 	}
 }
 
-// This is an acceptance test.
-// Requires the following env vars:
-// TEST_AWS_EC2_PKCS7
-// TEST_AWS_EC2_AMI_ID
-// TEST_AWS_EC2_ACCOUNT_ID
-// TEST_AWS_EC2_IAM_ROLE_ARN
-//
-// If the test is not being run on an EC2 instance that has access to
-// credentials using EC2RoleProvider, on top of the above vars, following
-// needs to be set:
-// TEST_AWS_SECRET_KEY
-// TEST_AWS_ACCESS_KEY
+/* This is an acceptance test.
+   Requires the following env vars:
+   TEST_AWS_EC2_PKCS7
+   TEST_AWS_EC2_IDENTITY_DOCUMENT
+   TEST_AWS_EC2_IDENTITY_DOCUMENT_SIG
+   TEST_AWS_EC2_AMI_ID
+   TEST_AWS_EC2_ACCOUNT_ID
+   TEST_AWS_EC2_IAM_ROLE_ARN
+
+   If this is being run on an EC2 instance, you can set the environment vars using this bash snippet:
+
+   export TEST_AWS_EC2_PKCS7=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/pkcs7)
+   export TEST_AWS_EC2_IDENTITY_DOCUMENT=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | base64 -w 0)
+   export TEST_AWS_EC2_IDENTITY_DOCUMENT_SIG=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/signature | tr -d '\n')
+   export TEST_AWS_EC2_AMI_ID=$(curl -s http://169.254.169.254/latest/meta-data/ami-id)
+   export TEST_AWS_EC2_IAM_ROLE_ARN=$(aws iam get-role --role-name $(curl -q http://169.254.169.254/latest/meta-data/iam/security-credentials/ -S -s) --query Role.Arn --output text)
+   export TEST_AWS_EC2_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+
+   If the test is not being run on an EC2 instance that has access to
+   credentials using EC2RoleProvider, on top of the above vars, following
+   needs to be set:
+   TEST_AWS_SECRET_KEY
+   TEST_AWS_ACCESS_KEY
+*/
 func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.T) {
 	// This test case should be run only when certain env vars are set and
 	// executed as an acceptance test.
@@ -983,6 +995,16 @@ func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.
 		t.Fatalf("env var TEST_AWS_EC2_PKCS7 not set")
 	}
 
+	identityDoc := os.Getenv("TEST_AWS_EC2_IDENTITY_DOCUMENT")
+	if identityDoc == "" {
+		t.Fatalf("env var TEST_AWS_EC2_IDENTITY_DOCUMENT not set")
+	}
+
+	identityDocSig := os.Getenv("TEST_AWS_EC2_IDENTITY_DOCUMENT_SIG")
+	if identityDocSig == "" {
+		t.Fatalf("env var TEST_AWS_EC2_IDENTITY_DOCUMENT_SIG not set")
+	}
+
 	amiID := os.Getenv("TEST_AWS_EC2_AMI_ID")
 	if amiID == "" {
 		t.Fatalf("env var TEST_AWS_EC2_AMI_ID not set")
@@ -1131,6 +1153,18 @@ func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.
 		t.Fatalf("bad: failed to login: resp:%#v\nerr:%v", resp, err)
 	}
 
+	// Attempt to re-login with the identity signture
+	delete(loginInput, "pkcs7")
+	loginInput["identity"] = identityDoc
+	loginInput["signature"] = identityDocSig
+	resp, err = b.HandleRequest(context.Background(), loginRequest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil || resp.Auth == nil || resp.IsError() {
+		t.Fatalf("bad: failed to login: resp:%#v\nerr:%v", resp, err)
+	}
+
 	// verify the presence of instance_id in the response object.
 	instanceID := resp.Auth.Metadata["instance_id"]
 	if instanceID == "" {

builtin/credential/cert/path_login.go

@@ -439,12 +439,28 @@ func validateConnState(roots *x509.CertPool, cs *tls.ConnectionState) ([][]*x509
 		}
 	}
 
-	chains, err := certs[0].Verify(opts)
-	if err != nil {
-		if _, ok := err.(x509.UnknownAuthorityError); ok {
-			return nil, nil
+	var chains [][]*x509.Certificate
+	var err error
+	switch {
+	case len(certs[0].DNSNames) > 0:
+		for _, dnsName := range certs[0].DNSNames {
+			opts.DNSName = dnsName
+			chains, err = certs[0].Verify(opts)
+			if err != nil {
+				if _, ok := err.(x509.UnknownAuthorityError); ok {
+					return nil, nil
+				}
+				return nil, errors.New("failed to verify client's certificate: " + err.Error())
+			}
+		}
+	default:
+		chains, err = certs[0].Verify(opts)
+		if err != nil {
+			if _, ok := err.(x509.UnknownAuthorityError); ok {
+				return nil, nil
+			}
+			return nil, errors.New("failed to verify client's certificate: " + err.Error())
 		}
-		return nil, errors.New("failed to verify client's certificate: " + err.Error())
 	}
 
 	return chains, nil

builtin/logical/pki/cert_util.go

@@ -1351,7 +1351,7 @@ func convertRespToPKCS8(resp *logical.Response) error {
 		return errwrap.Wrapf("error converting response to pkcs8: error parsing previous key: {{err}}", err)
 	}
 
-	keyData, err = x509.MarshalPKCS8PrivateKey(signer)
+	keyData, err = certutil.MarshalPKCS8PrivateKey(signer)
 	if err != nil {
 		return errwrap.Wrapf("error converting response to pkcs8: error marshaling pkcs8 key: {{err}}", err)
 	}

command/auth_enable.go

@@ -3,6 +3,7 @@ package command
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/vault/api"
 	"github.com/mitchellh/cli"
@@ -15,11 +16,13 @@ var _ cli.CommandAutocomplete = (*AuthEnableCommand)(nil)
 type AuthEnableCommand struct {
 	*BaseCommand
 
-	flagDescription string
-	flagPath        string
-	flagPluginName  string
-	flagLocal       bool
-	flagSealWrap    bool
+	flagDescription     string
+	flagPath            string
+	flagDefaultLeaseTTL time.Duration
+	flagMaxLeaseTTL     time.Duration
+	flagPluginName      string
+	flagLocal           bool
+	flagSealWrap        bool
 }
 
 func (c *AuthEnableCommand) Synopsis() string {
@@ -75,6 +78,24 @@ func (c *AuthEnableCommand) Flags() *FlagSets {
 			"\"/auth/<path>\".",
 	})
 
+	f.DurationVar(&DurationVar{
+		Name:       "default-lease-ttl",
+		Target:     &c.flagDefaultLeaseTTL,
+		Completion: complete.PredictAnything,
+		Usage: "The default lease TTL for this auth method. If unspecified, " +
+			"this defaults to the Vault server's globally configured default lease " +
+			"TTL.",
+	})
+
+	f.DurationVar(&DurationVar{
+		Name:       "max-lease-ttl",
+		Target:     &c.flagMaxLeaseTTL,
+		Completion: complete.PredictAnything,
+		Usage: "The maximum lease TTL for this auth method. If unspecified, " +
+			"this defaults to the Vault server's globally configured maximum lease " +
+			"TTL.",
+	})
+
 	f.StringVar(&StringVar{
 		Name:       "plugin-name",
 		Target:     &c.flagPluginName,
@@ -155,7 +176,9 @@ func (c *AuthEnableCommand) Run(args []string) int {
 		Local:       c.flagLocal,
 		SealWrap:    c.flagSealWrap,
 		Config: api.AuthConfigInput{
-			PluginName: c.flagPluginName,
+			DefaultLeaseTTL: c.flagDefaultLeaseTTL.String(),
+			MaxLeaseTTL:     c.flagMaxLeaseTTL.String(),
+			PluginName:      c.flagPluginName,
 		},
 	}); err != nil {
 		c.UI.Error(fmt.Sprintf("Error enabling %s auth: %s", authType, err))

command/server.go

@@ -86,6 +86,7 @@ type ServerCommand struct {
 	flagDevLeasedKV      bool
 	flagDevSkipInit      bool
 	flagDevThreeNode     bool
+	flagDevFourCluster   bool
 	flagDevTransactional bool
 	flagTestVerifyOnly   bool
 }
@@ -237,6 +238,13 @@ func (c *ServerCommand) Flags() *FlagSets {
 		Hidden:  true,
 	})
 
+	f.BoolVar(&BoolVar{
+		Name:    "dev-four-cluster",
+		Target:  &c.flagDevFourCluster,
+		Default: false,
+		Hidden:  true,
+	})
+
 	// TODO: should this be a public flag?
 	f.BoolVar(&BoolVar{
 		Name:    "test-verify-only",
@@ -295,10 +303,11 @@ func (c *ServerCommand) Run(args []string) int {
 	}
 	switch strings.ToLower(logFormat) {
 	case "vault", "vault_json", "vault-json", "vaultjson", "json", "":
-		if c.flagDevThreeNode {
+		if c.flagDevThreeNode || c.flagDevFourCluster {
 			c.logger = logbridge.NewLogger(hclog.New(&hclog.LoggerOptions{
 				Mutex:  &sync.Mutex{},
 				Output: c.logGate,
+				Level:  hclog.Trace,
 			})).LogxiLogger()
 		} else {
 			c.logger = logformat.NewVaultLoggerWithWriter(c.logGate, level)
@@ -313,7 +322,7 @@ func (c *ServerCommand) Run(args []string) int {
 	})
 
 	// Automatically enable dev mode if other dev flags are provided.
-	if c.flagDevHA || c.flagDevTransactional || c.flagDevLeasedKV || c.flagDevThreeNode {
+	if c.flagDevHA || c.flagDevTransactional || c.flagDevLeasedKV || c.flagDevThreeNode || c.flagDevFourCluster {
 		c.flagDev = true
 	}
 

helper/certutil/certutil_test.go

@@ -614,7 +614,7 @@ func setCerts() {
 			Bytes: marshaledKey,
 		}
 		privECKeyPem = string(pem.EncodeToMemory(keyPEMBlock))
-		marshaledKey, err = x509.MarshalPKCS8PrivateKey(key)
+		marshaledKey, err = MarshalPKCS8PrivateKey(key)
 		if err != nil {
 			panic(err)
 		}
@@ -680,7 +680,7 @@ func setCerts() {
 			Bytes: marshaledKey,
 		}
 		privRSAKeyPem = string(pem.EncodeToMemory(keyPEMBlock))
-		marshaledKey, err = x509.MarshalPKCS8PrivateKey(key)
+		marshaledKey, err = MarshalPKCS8PrivateKey(key)
 		if err != nil {
 			panic(err)
 		}