diff --git a/CHANGELOG.md b/CHANGELOG.md index 41f97336c504..00979390e536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ IMPROVEMENTS: * auth/aws: Allow using lists in role bind parameters [GH-3907] * server: Make sure `tls_disable_client_cert` is actually a true value rather than just set [GH-4049] + * storage/gcs: Allow specifying chunk size for transfers, which can reduce + memory utilization [GH-4060] * sys/capabilities: Add the ability to use multiple paths for capability checking [GH-3663] diff --git a/command/format.go b/command/format.go index d6e4171dfc3b..c547535eb6b8 100644 --- a/command/format.go +++ b/command/format.go @@ -325,7 +325,7 @@ func OutputSealStatus(ui cli.Ui, client *api.Client, status *api.SealStatusRespo // This is down here just to keep ordering consistent if showLeaderAddr { - out = append(out, fmt.Sprintf("Active Node Address: | %s", leaderStatus.LeaderAddress)) + out = append(out, fmt.Sprintf("Active Node Address | %s", leaderStatus.LeaderAddress)) } } } diff --git a/physical/gcs/gcs.go b/physical/gcs/gcs.go index c81e0489699f..85656d899ec1 100644 --- a/physical/gcs/gcs.go +++ b/physical/gcs/gcs.go @@ -21,9 +21,6 @@ import ( "google.golang.org/api/option" ) -// Verify GCSBackend satisfies the correct interfaces -var _ physical.Backend = (*GCSBackend)(nil) - // GCSBackend is a physical backend that stores data // within an Google Cloud Storage bucket. type GCSBackend struct { @@ -33,6 +30,15 @@ type GCSBackend struct { logger log.Logger } +var ( + // Verify GCSBackend satisfies the correct interfaces + _ physical.Backend = (*GCSBackend)(nil) + + // Number of bytes the writer will attempt to write in a single request. + // Defaults to 8Mb, as defined in the gcs library + chunkSize = 8 * 1024 * 1024 +) + // NewGCSBackend constructs a Google Cloud Storage backend using a pre-existing // bucket. Credentials can be provided to the backend, sourced // from environment variables or a service account file @@ -70,6 +76,18 @@ func NewGCSBackend(conf map[string]string, logger log.Logger) (physical.Backend, } } + chunkSizeStr, ok := conf["chunk_size"] + if ok { + chunkSize, err = strconv.Atoi(chunkSizeStr) + if err != nil { + return nil, errwrap.Wrapf("failed parsing chunk_size parameter: {{err}}", err) + } + chunkSize *= 1024 + if logger.IsDebug() { + logger.Debug("physical/gcs: chunk_size set", "chunk_size", chunkSize) + } + } + g := GCSBackend{ bucketName: bucketName, client: client, @@ -111,6 +129,7 @@ func (g *GCSBackend) Put(ctx context.Context, entry *physical.Entry) error { bucket := g.client.Bucket(g.bucketName) writer := bucket.Object(entry.Key).NewWriter(context.Background()) + writer.ChunkSize = chunkSize g.permitPool.Acquire() defer g.permitPool.Release() diff --git a/website/source/docs/auth/aws.html.md b/website/source/docs/auth/aws.html.md index a52d6fb76a69..a7cca908260b 100644 --- a/website/source/docs/auth/aws.html.md +++ b/website/source/docs/auth/aws.html.md @@ -514,7 +514,7 @@ time which is dynamically determined by three factors: `max_ttl` set on the role least of these three dictates the maximum TTL of the issued token, and correspondingly will be set as the expiration times of these entries. -The endpoints `aws/auth/tidy/identity-whitelist` and `aws/auth/tidy/roletag-blacklist` are +The endpoints `auth/aws/tidy/identity-whitelist` and `auth/aws/tidy/roletag-blacklist` are provided to clean up the entries present in these lists. These endpoints allow defining a safety buffer, such that an entry must not only be expired, but be past expiration by the amount of time dictated by the safety buffer in order diff --git a/website/source/docs/configuration/storage/google-cloud.html.md b/website/source/docs/configuration/storage/google-cloud.html.md index 2e6a98b66a74..7e8940c8e8e6 100644 --- a/website/source/docs/configuration/storage/google-cloud.html.md +++ b/website/source/docs/configuration/storage/google-cloud.html.md @@ -42,6 +42,13 @@ storage "gcs" { - `max_parallel` `(string: "128")` – Specifies the maximum number of concurrent requests. +- `chunk_size` `(string: "8192")` – Specifies the maximum kilobytes of each object + the gcs writer will attempt to send to the server in a single request. + If set to 0, it will attempt to send the whole object at once, but it will + not retry any failures either (not recommended). If you are not storing large + objects in Vault, it is recommended to set this to a low value (minimum is 256) + since it will drastically reduce the amount of memory Vault uses. + ## `gcs` Examples ### Default Example