Merge branch 'master' into cdcr-backend-grpc

* master: (85 commits) UI icon - add size (#6736) Add Priority Queue library to sdk (#6664) Add spellcheck="false" to form fields (#6744) Maximum typo in Vault UI (#6743) docs: Fix Markdown formatting error in AWS Auth (#6745) changelog++ Update OIDC Provider Setup docs (#6739) Update to use newer sdk Copy LogInput from audit package, add OptMarshaler interface (#6735) docs: fixed typo (#6732) Fix typo changelog++ Use Go modules in CircleCI (#6729) Fix recovery key backup path documentation Vendoring updated grpc Add link to R client on libraries list (#6722) UI ember engines (#6718) changelog++ Update grpc and protos (#6725) changelog++ ...
hashicorp · May 16, 2019 · da1b8cc · da1b8cc
2 parents 616e4e0 + 033b547
commit da1b8cc
Show file tree

Hide file tree

Showing 664 changed files with 8,497 additions and 4,083 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,204 @@
+version: 2
+
+references:
+  images:
+    go: &GOLANG_IMAGE golang:1.12.4-stretch  # Pin Go to patch version (ex: 1.2.3)
+    node: &NODE_IMAGE node:10-stretch  # Pin Node.js to major version (ex: 10)
+
+  environment: &ENVIRONMENT
+    CIRCLECI_CLI_VERSION: 0.1.5546  # Pin CircleCI CLI to patch version (ex: 1.2.3)
+    GO_VERSION: 1.12.4  # Pin Go to patch version (ex: 1.2.3)
+    GOTESTSUM_VERSION: 0.3.3  # Pin gotestsum to patch version (ex: 1.2.3)
+
+  cache:
+    go-sum: &GO_SUM_CACHE_KEY go-sum-v1-{{ checksum "go.sum" }}
+    yarn-lock: &YARN_LOCK_CACHE_KEY yarn-lock-v1-{{ checksum "ui/yarn.lock" }}
+
+jobs:
+  install-ui-dependencies:
+    docker:
+      - image: *NODE_IMAGE
+    working_directory: /src
+    steps:
+      - checkout
+      - restore_cache:
+          key: *YARN_LOCK_CACHE_KEY
+      - run:
+          name: Install UI dependencies
+          command: |
+            set -eux -o pipefail
+
+            cd ui
+            yarn install --ignore-optional
+            npm rebuild node-sass
+      - save_cache:
+          key: *YARN_LOCK_CACHE_KEY
+          paths:
+            - ui/node_modules
+
+  go-mod-download:
+    docker:
+      - image: *GOLANG_IMAGE
+    working_directory: /src
+    steps:
+      - checkout
+      - restore_cache:
+          key: *GO_SUM_CACHE_KEY
+      - run:
+          name: Download Go modules
+          command: go mod download
+      - run:
+          name: Verify checksums of Go modules
+          command: go mod verify
+      - save_cache:
+          key: *GO_SUM_CACHE_KEY
+          paths:
+            - /go/pkg/mod
+
+  build-go-dev:
+    docker:
+      - image: *GOLANG_IMAGE
+    working_directory: /src
+    steps:
+      - checkout
+      - restore_cache:
+          key: *GO_SUM_CACHE_KEY
+      - run:
+          name: Build dev binary
+          command: |
+            set -eux -o pipefail
+
+            # Move dev UI assets to expected location
+            rm -rf ./pkg
+            mkdir ./pkg
+
+            # Build dev binary
+            make bootstrap dev
+      - persist_to_workspace:
+          root: .
+          paths:
+            - bin
+
+  test-ui:
+    docker:
+      - image: *NODE_IMAGE
+    working_directory: /src
+    resource_class: medium+
+    steps:
+      - checkout
+      - restore_cache:
+          key: *YARN_LOCK_CACHE_KEY
+      - attach_workspace:
+          at: .
+      - run:
+          name: Test UI
+          command: |
+            set -eux -o pipefail
+
+            # Install Chrome
+            wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub \
+              | apt-key add -
+            echo "deb http://dl.google.com/linux/chrome/deb/ stable main" \
+              | tee /etc/apt/sources.list.d/google-chrome.list
+            apt-get update
+            apt-get -y install google-chrome-stable
+            rm /etc/apt/sources.list.d/google-chrome.list
+            rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+            # Add ./bin to the PATH so vault binary can be run by Ember tests
+            export PATH="${PWD}"/bin:${PATH}
+
+            # Run Ember tests
+            cd ui
+            mkdir -p test-results/qunit
+            yarn run test-oss
+      - store_artifacts:
+          path: ui/test-results
+      - store_test_results:
+          path: ui/test-results
+
+  test-go:
+    machine: true
+    environment:
+      <<: *ENVIRONMENT
+      GO_TAGS:
+    parallelism: 2
+    working_directory: ~/src
+    steps:
+      - checkout
+      - run:
+          name: Allow circleci user to restore Go modules cache
+          command: |
+            set -eux -o pipefail
+
+            sudo mkdir /go
+            sudo chown -R circleci:circleci /go
+      - restore_cache:
+          key: *GO_SUM_CACHE_KEY
+      - run:
+          name: Run Go tests
+          command: |
+            set -eux -o pipefail
+
+            # Install Go
+            curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz"
+            sudo rm -rf /usr/local/go
+            sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz"
+            rm -f "go${GO_VERSION}.linux-amd64.tar.gz"
+            export GOPATH=/go
+            export PATH="${PATH}:${GOPATH}/bin:/usr/local/go/bin"
+
+            # Install CircleCI CLI
+            curl -sSL \
+                "https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CIRCLECI_CLI_VERSION}/circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64.tar.gz" \
+              | sudo tar --overwrite -xz \
+                -C /usr/local/bin \
+                "circleci-cli_${CIRCLECI_CLI_VERSION}_linux_amd64/circleci"
+
+            # Split Go tests by prior test times
+            package_names=$(go list \
+                -tags "${GO_TAGS}" \
+                ./... \
+              | grep -v /vendor/ \
+              | sort \
+              | circleci tests split --split-by=timings --timings-type=classname)
+
+            # Install gotestsum
+            curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz" \
+              | sudo tar --overwrite -xz -C /usr/local/bin gotestsum
+
+            # Run tests
+            make prep
+            mkdir -p test-results/go-test
+            CGO_ENABLED= \
+              VAULT_ADDR= \
+              VAULT_TOKEN= \
+              VAULT_DEV_ROOT_TOKEN_ID= \
+              VAULT_ACC= \
+              gotestsum --format=short-verbose --junitfile test-results/go-test/results.xml -- \
+                -tags "${GO_TAGS}" \
+                -timeout=40m \
+                -parallel=20 \
+                ${package_names}
+      - store_artifacts:
+          path: test-results
+      - store_test_results:
+          path: test-results
+
+workflows:
+  version: 2
+
+  ci:
+    jobs:
+      - install-ui-dependencies
+      - go-mod-download
+      - build-go-dev:
+          requires:
+            - go-mod-download
+      - test-ui:
+          requires:
+            - install-ui-dependencies
+            - build-go-dev
+      - test-go:
+          requires:
+            - build-go-dev
diff --git a/.travis.yml b/.travis.yml
@@ -39,6 +39,7 @@ branches:
 
 env:
   - TEST_COMMAND='make dev test-ember'
+  - TEST_COMMAND='make dev ember-ci-test'
   - TEST_COMMAND='travis_wait 75 make testtravis'
   - TEST_COMMAND='travis_wait 75 make testracetravis'
   - GO111MODULE=on

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,18 +5,50 @@ CHANGES:
  * autoseal/aws: The user-configured regions on the AWSKMS seal stanza 
    will now be preferred over regions set in the enclosing environment.
    This is a _breaking_ change.
+ * audit: Several more values in audit logs now are omitted if they are empty.
+   This helps reduce the size of audit log entries by not reproducing keys in
+   each entry that commonly don't contain any value, which can help in cases
+   where audit log entries are above the maximum UDP packet size and others.
+   See [GH-6387](https://github.com/hashicorp/vault/pull/6387) for details.
+ * backends: both PeriodicFunc and WALRollback functions will be called if 
+   both are provided. Previously WALRollback would only be called if PeriodicFunc 
+   was not set. See [GH-6717](https://github.com/hashicorp/vault/pull/6717) for 
+   details.
+ * Go Modules change: Vault now uses Go Modules to manage dependencies. As a
+   result to both reduce transitive dependencies for API library users and
+   plugin authors, and to work around various conflicts, we have moved various
+   helpers around, mostly under an `sdk/` submodule. A couple of functions have
+   also moved from plugin helper code to the `api/` submodule. If you are a
+   plugin author, take a look at some of our official plugins and the paths
+   they are importing for guidance.
+
+FEATURES:
+
+ * storage/postgres: Add HA support for PostgreSQL versions >= 9.5 [GH-5731]
 
 IMPROVEMENTS: 
 
+ * auth/jwt: A JWKS endpoint may now be configured for signature verification [JWT-43]
+ * auth/jwt: `bound_claims` will now match received claims that are lists if any element
+   of the list is one of the expected values [JWT-50]
  * ui: KV v1 and v2 will now gracefully degrade allowing a write without read
    workflow in the UI [GH-6570]
-
+ * storage/postgres: LIST now performs better on large datasets. [GH-6546]
+
 BUG FIXES: 
 
+ * auth/jwt: Fix bound constraint checking so `bound_claims` satisfies the requirement [JWT-49]
+ * auth/okta: Fix handling of group names containing slashes [GH-6665]
  * core: Correctly honor non-HMAC request keys when auditing requests [GH-6653]
  * core: Fix the `x-vault-unauthenticated` value in OpenAPI for a number of endpoints [GH-6654]
+ * core: Fix issue where some OpenAPI parameters were incorrectly listed as being sent
+   as a header [GH-6679]
+ * pki: fix a panic when a client submits a null value [GH-5679]
  * replication: Fix an issue causing startup problems if a namespace policy
    wasn't replicated properly
+ * storage/consul: recognize `https://` address even if schema not specified [GH-6602]
+ * storage/dynamodb: Fix an issue where a deleted lock key in DynamoDB (HA) could cause
+   constant switching of the active node [GH-6637]
  * storage/dynamodb: Eliminate a high-CPU condition that could occur if an error was
    received from the DynamoDB API [GH-6640]
  * replication: Properly update mount entry cache on a secondary to apply all
@@ -174,7 +206,7 @@ IMPROVEMENTS:
  * core/metrics: Prometheus pull support using a new sys/metrics endpoint. [GH-5308]
  * core: On non-windows platforms a SIGUSR2 will make the server log a dump of
    all running goroutines' stack traces for debugging purposes [GH-6240]
- * replication: The inital replication indexing process on newly initialized or upgraded
+ * replication: The initial replication indexing process on newly initialized or upgraded
    clusters now runs asynchronously
  * sentinel: Add token namespace id and path, available in rules as 
    token.namespace.id and token.namespace.path

diff --git a/Makefile b/Makefile
@@ -128,6 +128,12 @@ test-ember:
 	@echo "--> Running ember tests"
 	@cd ui && yarn run test-oss
 
+ember-ci-test:
+	@echo "--> Installing JavaScript assets"
+	@cd ui && yarn --ignore-optional
+	@echo "--> Running ember tests in Browserstack"
+	@cd ui && yarn run test:browserstack
+
 ember-dist:
 	@echo "--> Installing JavaScript assets"
 	@cd ui && yarn --ignore-optional

diff --git a/README.md b/README.md
@@ -63,9 +63,11 @@ first need [Go](https://www.golang.org) installed on your machine (version
 1.12.1+ is *required*).
 
 For local dev first make sure Go is properly installed, including setting up a
-[GOPATH](https://golang.org/doc/code.html#GOPATH). Next, clone this repository
-into `$GOPATH/src/github.com/hashicorp/vault`. You can then download any
-required build tools by bootstrapping your environment:
+[GOPATH](https://golang.org/doc/code.html#GOPATH). Ensure that `$GOPATH/bin` is in
+your path as some distributions bundle old version of build tools. Next, clone this
+repository. Vault uses [Go Modules](https://github.com/golang/go/wiki/Modules),
+so it is recommended that you clone the repository ***outside*** of the GOPATH.
+You can then download any required build tools by bootstrapping your environment:
 
 ```sh
 $ make bootstrap

diff --git a/api/plugin_helpers.go b/api/plugin_helpers.go
@@ -12,7 +12,16 @@ import (
 	squarejwt "gopkg.in/square/go-jose.v2/jwt"
 
 	"github.com/hashicorp/errwrap"
-	"github.com/hashicorp/vault/sdk/helper/pluginutil"
+)
+
+var (
+	// PluginMetadataModeEnv is an ENV name used to disable TLS communication
+	// to bootstrap mounting plugins.
+	PluginMetadataModeEnv = "VAULT_PLUGIN_METADATA_MODE"
+
+	// PluginUnwrapTokenEnv is the ENV name used to pass unwrap tokens to the
+	// plugin.
+	PluginUnwrapTokenEnv = "VAULT_UNWRAP_TOKEN"
 )
 
 // PluginAPIClientMeta is a helper that plugins can use to configure TLS connections
@@ -61,12 +70,12 @@ func (f *PluginAPIClientMeta) GetTLSConfig() *TLSConfig {
 // VaultPluginTLSProvider is run inside a plugin and retrieves the response
 // wrapped TLS certificate from vault. It returns a configured TLS Config.
 func VaultPluginTLSProvider(apiTLSConfig *TLSConfig) func() (*tls.Config, error) {
-	if os.Getenv(pluginutil.PluginMetadataModeEnv) == "true" {
+	if os.Getenv(PluginMetadataModeEnv) == "true" {
 		return nil
 	}
 
 	return func() (*tls.Config, error) {
-		unwrapToken := os.Getenv(pluginutil.PluginUnwrapTokenEnv)
+		unwrapToken := os.Getenv(PluginUnwrapTokenEnv)
 
 		parsedJWT, err := squarejwt.ParseSigned(unwrapToken)
 		if err != nil {

diff --git a/builtin/credential/ldap/path_groups.go b/builtin/credential/ldap/path_groups.go
@@ -26,12 +26,12 @@ func pathGroups(b *backend) *framework.Path {
 	return &framework.Path{
 		Pattern: `groups/(?P<name>.+)`,
 		Fields: map[string]*framework.FieldSchema{
-			"name": &framework.FieldSchema{
+			"name": {
 				Type:        framework.TypeString,
 				Description: "Name of the LDAP group.",
 			},
 
-			"policies": &framework.FieldSchema{
+			"policies": {
 				Type:        framework.TypeCommaStringSlice,
 				Description: "Comma-separated list of policies associated to the group.",
 			},
@@ -132,17 +132,14 @@ func (b *backend) pathGroupWrite(ctx context.Context, req *logical.Request, d *f
 }
 
 func (b *backend) pathGroupList(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
-	keys, err := logical.CollectKeys(ctx, req.Storage)
+	keys, err := logical.CollectKeysWithPrefix(ctx, req.Storage, "group/")
 	if err != nil {
 		return nil, err
 	}
-	retKeys := make([]string, 0)
-	for _, key := range keys {
-		if strings.HasPrefix(key, "group/") && !strings.HasPrefix(key, "/") {
-			retKeys = append(retKeys, strings.TrimPrefix(key, "group/"))
-		}
+	for i := range keys {
+		keys[i] = strings.TrimPrefix(keys[i], "group/")
 	}
-	return logical.ListResponse(retKeys), nil
+	return logical.ListResponse(keys), nil
 }
 
 type GroupEntry struct {

diff --git a/builtin/credential/ldap/path_users.go b/builtin/credential/ldap/path_users.go
@@ -148,18 +148,14 @@ func (b *backend) pathUserWrite(ctx context.Context, req *logical.Request, d *fr
 }
 
 func (b *backend) pathUserList(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
-	keys, err := logical.CollectKeys(ctx, req.Storage)
+	keys, err := logical.CollectKeysWithPrefix(ctx, req.Storage, "user/")
 	if err != nil {
 		return nil, err
 	}
-	retKeys := make([]string, 0)
-	for _, key := range keys {
-		if strings.HasPrefix(key, "user/") && !strings.HasPrefix(key, "/") {
-			retKeys = append(retKeys, strings.TrimPrefix(key, "user/"))
-		}
+	for i := range keys {
+		keys[i] = strings.TrimPrefix(keys[i], "user/")
 	}
-	return logical.ListResponse(retKeys), nil
-
+	return logical.ListResponse(keys), nil
 }
 
 type UserEntry struct {