Merge branch 'master' into ui/replication-status-discoverability

CHANGELOG.md

@@ -10,12 +10,14 @@ CHANGES:
 * plugin: Add SDK method, `Sys.ReloadPlugin`, and CLI command, `vault plugin reload`, 
   for reloading plugins. [[GH-8777](https://github.com/hashicorp/vault/pull/8777)]
 * sentinel: Add a sentinel config section, and "additional_enabled_modules", a list of Sentinel modules that may be imported in addition to the defaults.
+* cubbyhole: Reject reads and writes to an empty ("") path. [[GH-8971](https://github.com/hashicorp/vault/pull/8971)]
 
 IMPROVEMENTS:
 
 * secrets/gcp: Support BigQuery dataset ACLs in absence of IAM endpoints [[GH-78](https://github.com/hashicorp/vault-plugin-secrets-gcp/pull/78)]
 * sdk/framework: Support accepting TypeFloat parameters over the API [[GH-8923](https://github.com/hashicorp/vault/pull/8923)]
 * ui: Update TTL picker styling on SSH secret engine [[GH-8891](https://github.com/hashicorp/vault/pull/8891)]
+* ui: Only render the JWT input field of the Vault login form on mounts configured for JWT auth [[GH-8952](https://github.com/hashicorp/vault/pull/8952)]
 
 BUG FIXES:
 
@@ -30,7 +32,16 @@ IMPROVEMENTS:
 * storage/raft: The storage stanza now accepts `leader_ca_cert_file`, `leader_client_cert_file`, and 
   `leader_client_key_file` parameters to read and parse TLS certificate information from paths on disk.
   Existing non-path based parameters will continue to work, but their values will need to be provided as a 
-  single-line string with newlines delimited by `\n`.
+  single-line string with newlines delimited by `\n`.  [[GH-8894](https://github.com/hashicorp/vault/pull/8894)]
+
+BUG FIXES:
+
+* serviceregistration: Fix a regression for Consul service registration that ignored using the listener address as
+  the redirect address unless api_addr was provided. It now properly uses the same redirect address as the one
+  used by Vault's Core object. [[GH-8976](https://github.com/hashicorp/vault/pull/8976)] 
+* sys: The path provided in `sys/internal/ui/mounts/:path` is now namespace-aware. This fixes an issue
+  with `vault kv` subcommands that had namespaces provided in the path returning permission denied all the time.
+  [[GH-8962](https://github.com/hashicorp/vault/pull/8962)]
 
 ## 1.4.1 (April 30th, 2020)
 
@@ -167,6 +178,7 @@ BUG FIXES:
 * identity: Fix incorrect caching of identity token JWKS responses [[GH-8412](https://github.com/hashicorp/vault/pull/8412)]
 * metrics/stackdriver: Fix issue that prevents the stackdriver metrics library to create unnecessary stackdriver descriptors [[GH-8073](https://github.com/hashicorp/vault/pull/8073)]
 * replication: Fix issue causing cubbyholes in namespaces on performance secondaries to not work.
+* replication (enterprise): Unmounting a dynamic secrets backend could sometimes lead to replication errors.  Change the order of operations to prevent that.
 * seal (enterprise): Fix seal migration when transactional seal wrap backend is in use.
 * secrets/database/influxdb: Fix potential panic if connection to the InfluxDB database cannot be established [[GH-8282](https://github.com/hashicorp/vault/pull/8282)]
 * secrets/database/mysql: Ensures default static credential rotation statements are used [[GH-8240](https://github.com/hashicorp/vault/pull/8240)]

Makefile

@@ -134,10 +134,7 @@ ci-bootstrap:
 
 # bootstrap the build by downloading additional tools that may be used by devs
 bootstrap: ci-bootstrap
-	@for tool in  $(EXTERNAL_TOOLS) ; do \
-		echo "Installing/Updating $$tool" ; \
-		GO111MODULE=off $(GO_CMD) get -u $$tool; \
-	done
+	go generate -tags tools tools/tools.go
 
 # Note: if you have plugins in GOPATH you can update all of them via something like:
 # for i in $(ls | grep vault-plugin-); do cd $i; git remote update; git reset --hard origin/master; dep ensure -update; git add .; git commit; git push; cd ..; done

api/go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.6.2
 	github.com/hashicorp/go-rootcerts v1.0.1
 	github.com/hashicorp/hcl v1.0.0
-	github.com/hashicorp/vault/sdk v0.1.14-0.20200215195600-2ca765f0a500
+	github.com/hashicorp/vault/sdk v0.1.14-0.20200514144402-4bfac290c352
 	github.com/mitchellh/mapstructure v1.1.2
 	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4

builtin/credential/okta/path_config.go

@@ -3,12 +3,12 @@ package okta
 import (
 	"context"
 	"fmt"
-	"github.com/hashicorp/go-cleanhttp"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-cleanhttp"
 	oktaold "github.com/chrismalek/oktasdk-go/okta"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/helper/tokenutil"

builtin/logical/database/rollback_test.go

@@ -2,12 +2,13 @@ package database
 
 import (
 	"context"
+	"strings"
+	"testing"
+
 	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/sdk/database/dbplugin"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/logical"
-	"strings"
-	"testing"
 )
 
 const (

builtin/logical/pki/path_config_ca.go

@@ -47,11 +47,14 @@ func (b *backend) pathCAWrite(ctx context.Context, req *logical.Request, data *f
 		}
 	}
 
-	if parsedBundle.PrivateKey == nil ||
-		parsedBundle.PrivateKeyType == certutil.UnknownPrivateKey {
+	if parsedBundle.PrivateKey == nil {
 		return logical.ErrorResponse("private key not found in the PEM bundle"), nil
 	}
 
+	if parsedBundle.PrivateKeyType == certutil.UnknownPrivateKey {
+		return logical.ErrorResponse("unknown private key found in the PEM bundle"), nil
+	}
+
 	if parsedBundle.Certificate == nil {
 		return logical.ErrorResponse("no certificate found in the PEM bundle"), nil
 	}

command/agent.go

@@ -482,15 +482,8 @@ func (c *AgentCommand) Run(args []string) int {
 			// Parse 'require_request_header' listener config option, and wrap
 			// the request handler if necessary
 			muxHandler := cacheHandler
-			if v, ok := lnConfig.Config[agentConfig.RequireRequestHeader]; ok {
-				switch v {
-				case true:
-					muxHandler = verifyRequestHeader(muxHandler)
-				case false /* noop */ :
-				default:
-					c.UI.Error(fmt.Sprintf("Invalid value for 'require_request_header': %v", v))
-					return 1
-				}
+			if lnConfig.RequireRequestHeader {
+				muxHandler = verifyRequestHeader(muxHandler)
 			}
 
 			// Create a muxer and add paths relevant for the lease cache layer

command/agent/cache/listener.go

@@ -7,16 +7,13 @@ import (
 
 	"strings"
 
-	"github.com/hashicorp/vault/command/agent/config"
 	"github.com/hashicorp/vault/command/server"
+	"github.com/hashicorp/vault/internalshared/configutil"
 	"github.com/hashicorp/vault/internalshared/listenerutil"
 )
 
-func StartListener(lnConfig *config.Listener) (net.Listener, *tls.Config, error) {
-	addr, ok := lnConfig.Config["address"].(string)
-	if !ok {
-		return nil, nil, fmt.Errorf("invalid address")
-	}
+func StartListener(lnConfig *configutil.Listener) (net.Listener, *tls.Config, error) {
+	addr := lnConfig.Address
 
 	var ln net.Listener
 	var err error
@@ -41,13 +38,13 @@ func StartListener(lnConfig *config.Listener) (net.Listener, *tls.Config, error)
 
 	case "unix":
 		var uConfig *listenerutil.UnixSocketsConfig
-		if lnConfig.Config["socket_mode"] != nil &&
-			lnConfig.Config["socket_user"] != nil &&
-			lnConfig.Config["socket_group"] != nil {
+		if lnConfig.SocketMode != "" &&
+			lnConfig.SocketUser != "" &&
+			lnConfig.SocketGroup != "" {
 			uConfig = &listenerutil.UnixSocketsConfig{
-				Mode:  lnConfig.Config["socket_mode"].(string),
-				User:  lnConfig.Config["socket_user"].(string),
-				Group: lnConfig.Config["socket_group"].(string),
+				Mode:  lnConfig.SocketMode,
+				User:  lnConfig.SocketUser,
+				Group: lnConfig.SocketGroup,
 			}
 		}
 		ln, err = listenerutil.UnixSocketListener(addr, uConfig)
@@ -60,10 +57,13 @@ func StartListener(lnConfig *config.Listener) (net.Listener, *tls.Config, error)
 	}
 
 	props := map[string]string{"addr": ln.Addr().String()}
-	ln, props, _, tlsConf, err := listenerutil.WrapTLS(ln, props, lnConfig.Config, nil)
+	tlsConf, _, err := listenerutil.TLSConfig(lnConfig, props, nil)
 	if err != nil {
 		return nil, nil, err
 	}
+	if tlsConf != nil {
+		ln = tls.NewListener(ln, tlsConf)
+	}
 
 	return ln, tlsConf, nil
 }

command/agent/cache/testing.go

@@ -72,11 +72,11 @@ type mockTokenVerifierProxier struct {
 func (p *mockTokenVerifierProxier) Send(ctx context.Context, req *SendRequest) (*SendResponse, error) {
 	p.currentToken = req.Token
 	resp := newTestSendResponse(http.StatusOK,
-		`{"data": {"id": "` + p.currentToken + `"}}`)
+		`{"data": {"id": "`+p.currentToken+`"}}`)
 
 	return resp, nil
 }
 
-func (p *mockTokenVerifierProxier) GetCurrentRequestToken() (string) {
+func (p *mockTokenVerifierProxier) GetCurrentRequestToken() string {
 	return p.currentToken
-}
+}

command/agent/config/config.go

@@ -14,16 +14,17 @@ import (
 	"github.com/hashicorp/hcl"
 	"github.com/hashicorp/hcl/hcl/ast"
 	"github.com/hashicorp/vault/helper/namespace"
+	"github.com/hashicorp/vault/internalshared/configutil"
 	"github.com/hashicorp/vault/sdk/helper/parseutil"
 	"github.com/mitchellh/mapstructure"
 )
 
 // Config is the configuration for the vault server.
 type Config struct {
+	*configutil.SharedConfig `hcl:"-"`
+
 	AutoAuth      *AutoAuth                  `hcl:"auto_auth"`
 	ExitAfterAuth bool                       `hcl:"exit_after_auth"`
-	PidFile       string                     `hcl:"pid_file"`
-	Listeners     []*Listener                `hcl:"listeners"`
 	Cache         *Cache                     `hcl:"cache"`
 	Vault         *Vault                     `hcl:"vault"`
 	Templates     []*ctconfig.TemplateConfig `hcl:"templates"`
@@ -48,15 +49,6 @@ type Cache struct {
 	ForceAutoAuthToken  bool        `hcl:"-"`
 }
 
-// Listener contains configuration for any Vault Agent listeners
-type Listener struct {
-	Type   string
-	Config map[string]interface{}
-}
-
-// RequireRequestHeader is a listener configuration option
-const RequireRequestHeader = "require_request_header"
-
 // AutoAuth is the configured authentication method and sinks
 type AutoAuth struct {
 	Method *Method `hcl:"-"`
@@ -89,6 +81,12 @@ type Sink struct {
 	Config     map[string]interface{}
 }
 
+func NewConfig() *Config {
+	return &Config{
+		SharedConfig: new(configutil.SharedConfig),
+	}
+}
+
 // LoadConfig loads the configuration at the given path, regardless if
 // its a file or directory.
 func LoadConfig(path string) (*Config, error) {
@@ -114,29 +112,31 @@ func LoadConfig(path string) (*Config, error) {
 	}
 
 	// Start building the result
-	var result Config
-	if err := hcl.DecodeObject(&result, obj); err != nil {
+	result := NewConfig()
+	if err := hcl.DecodeObject(result, obj); err != nil {
 		return nil, err
 	}
 
+	sharedConfig, err := configutil.ParseConfig(string(d))
+	if err != nil {
+		return nil, err
+	}
+	result.SharedConfig = sharedConfig
+
 	list, ok := obj.Node.(*ast.ObjectList)
 	if !ok {
 		return nil, fmt.Errorf("error parsing: file doesn't contain a root object")
 	}
 
-	if err := parseAutoAuth(&result, list); err != nil {
+	if err := parseAutoAuth(result, list); err != nil {
 		return nil, errwrap.Wrapf("error parsing 'auto_auth': {{err}}", err)
 	}
 
-	if err := parseListeners(&result, list); err != nil {
-		return nil, errwrap.Wrapf("error parsing 'listeners': {{err}}", err)
-	}
-
-	if err := parseCache(&result, list); err != nil {
+	if err := parseCache(result, list); err != nil {
 		return nil, errwrap.Wrapf("error parsing 'cache':{{err}}", err)
 	}
 
-	if err := parseTemplates(&result, list); err != nil {
+	if err := parseTemplates(result, list); err != nil {
 		return nil, errwrap.Wrapf("error parsing 'template': {{err}}", err)
 	}
 
@@ -161,12 +161,12 @@ func LoadConfig(path string) (*Config, error) {
 		}
 	}
 
-	err = parseVault(&result, list)
+	err = parseVault(result, list)
 	if err != nil {
 		return nil, errwrap.Wrapf("error parsing 'vault':{{err}}", err)
 	}
 
-	return &result, nil
+	return result, nil
 }
 
 func parseVault(result *Config, list *ast.ObjectList) error {
@@ -245,47 +245,6 @@ func parseCache(result *Config, list *ast.ObjectList) error {
 	return nil
 }
 
-func parseListeners(result *Config, list *ast.ObjectList) error {
-	name := "listener"
-
-	listenerList := list.Filter(name)
-
-	var listeners []*Listener
-	for _, item := range listenerList.Items {
-		var lnConfig map[string]interface{}
-		err := hcl.DecodeObject(&lnConfig, item.Val)
-		if err != nil {
-			return err
-		}
-
-		var lnType string
-		switch {
-		case lnConfig["type"] != nil:
-			lnType = lnConfig["type"].(string)
-			delete(lnConfig, "type")
-		case len(item.Keys) == 1:
-			lnType = strings.ToLower(item.Keys[0].Token.Value().(string))
-		default:
-			return errors.New("listener type must be specified")
-		}
-
-		switch lnType {
-		case "unix", "tcp":
-		default:
-			return fmt.Errorf("invalid listener type %q", lnType)
-		}
-
-		listeners = append(listeners, &Listener{
-			Type:   lnType,
-			Config: lnConfig,
-		})
-	}
-
-	result.Listeners = listeners
-
-	return nil
-}
-
 func parseAutoAuth(result *Config, list *ast.ObjectList) error {
 	name := "auto_auth"