Merge remote-tracking branch 'origin/master' into auth_aws_no_prefix_…

…matching

CHANGELOG.md

@@ -12,6 +12,8 @@ IMPROVEMENTS:
  * auth/aws: Allow using lists in role bind parameters [GH-3907]
  * server: Make sure `tls_disable_client_cert` is actually a true value rather
    than just set [GH-4049]
+ * storage/gcs: Allow specifying chunk size for transfers, which can reduce
+   memory utilization [GH-4060]
  * sys/capabilities: Add the ability to use multiple paths for capability
    checking [GH-3663]
 

command/format.go

@@ -325,7 +325,7 @@ func OutputSealStatus(ui cli.Ui, client *api.Client, status *api.SealStatusRespo
 
 			// This is down here just to keep ordering consistent
 			if showLeaderAddr {
-				out = append(out, fmt.Sprintf("Active Node Address: | %s", leaderStatus.LeaderAddress))
+				out = append(out, fmt.Sprintf("Active Node Address | %s", leaderStatus.LeaderAddress))
 			}
 		}
 	}

physical/gcs/gcs.go

@@ -21,9 +21,6 @@ import (
 	"google.golang.org/api/option"
 )
 
-// Verify GCSBackend satisfies the correct interfaces
-var _ physical.Backend = (*GCSBackend)(nil)
-
 // GCSBackend is a physical backend that stores data
 // within an Google Cloud Storage bucket.
 type GCSBackend struct {
@@ -33,6 +30,15 @@ type GCSBackend struct {
 	logger     log.Logger
 }
 
+var (
+	// Verify GCSBackend satisfies the correct interfaces
+	_ physical.Backend = (*GCSBackend)(nil)
+
+	// Number of bytes the writer will attempt to write in a single request.
+	// Defaults to 8Mb, as defined in the gcs library
+	chunkSize = 8 * 1024 * 1024
+)
+
 // NewGCSBackend constructs a Google Cloud Storage backend using a pre-existing
 // bucket. Credentials can be provided to the backend, sourced
 // from environment variables or a service account file
@@ -70,6 +76,18 @@ func NewGCSBackend(conf map[string]string, logger log.Logger) (physical.Backend,
 		}
 	}
 
+	chunkSizeStr, ok := conf["chunk_size"]
+	if ok {
+		chunkSize, err = strconv.Atoi(chunkSizeStr)
+		if err != nil {
+			return nil, errwrap.Wrapf("failed parsing chunk_size parameter: {{err}}", err)
+		}
+		chunkSize *= 1024
+		if logger.IsDebug() {
+			logger.Debug("physical/gcs: chunk_size set", "chunk_size", chunkSize)
+		}
+	}
+
 	g := GCSBackend{
 		bucketName: bucketName,
 		client:     client,
@@ -111,6 +129,7 @@ func (g *GCSBackend) Put(ctx context.Context, entry *physical.Entry) error {
 
 	bucket := g.client.Bucket(g.bucketName)
 	writer := bucket.Object(entry.Key).NewWriter(context.Background())
+	writer.ChunkSize = chunkSize
 
 	g.permitPool.Acquire()
 	defer g.permitPool.Release()

website/source/docs/auth/aws.html.md

@@ -514,7 +514,7 @@ time which is dynamically determined by three factors: `max_ttl` set on the role
 least of these three dictates the maximum TTL of the issued token, and
 correspondingly will be set as the expiration times of these entries.
 
-The endpoints `aws/auth/tidy/identity-whitelist` and `aws/auth/tidy/roletag-blacklist` are
+The endpoints `auth/aws/tidy/identity-whitelist` and `auth/aws/tidy/roletag-blacklist` are
 provided to clean up the entries present in these lists. These endpoints allow
 defining a safety buffer, such that an entry must not only be expired, but be
 past expiration by the amount of time dictated by the safety buffer in order

website/source/docs/configuration/storage/google-cloud.html.md

@@ -42,6 +42,13 @@ storage "gcs" {
 - `max_parallel` `(string: "128")` – Specifies the maximum number of concurrent
   requests.
 
+- `chunk_size` `(string: "8192")` – Specifies the maximum kilobytes of each object
+  the gcs writer will attempt to send to the server in a single request.
+  If set to 0, it will attempt to send the whole object at once, but it will
+  not retry any failures either (not recommended). If you are not storing large
+  objects in Vault, it is recommended to set this to a low value (minimum is 256)
+  since it will drastically reduce the amount of memory Vault uses.
+
 ## `gcs` Examples
 
 ### Default Example