From 884e6980c0aeff25c93e795203436dab71528f4f Mon Sep 17 00:00:00 2001 From: Dan Everton Date: Fri, 10 Mar 2017 13:14:52 +1000 Subject: [PATCH 1/4] Add permitPool support to S3 --- physical/s3.go | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/physical/s3.go b/physical/s3.go index 3e6809c96e9b..867a9b7c11e4 100644 --- a/physical/s3.go +++ b/physical/s3.go @@ -25,6 +25,7 @@ type S3Backend struct { bucket string client *s3.S3 logger log.Logger + permitPool *PermitPool } // newS3Backend constructs a S3 backend using a pre-existing @@ -85,10 +86,23 @@ func newS3Backend(conf map[string]string, logger log.Logger) (Backend, error) { return nil, fmt.Errorf("unable to access bucket '%s': %v", bucket, err) } + maxParStr, ok := conf["max_parallel"] + var maxParInt int + if ok { + maxParInt, err = strconv.Atoi(maxParStr) + if err != nil { + return nil, errwrap.Wrapf("failed parsing max_parallel parameter: {{err}}", err) + } + if logger.IsDebug() { + logger.Debug("s3: max_parallel set", "max_parallel", maxParInt) + } + } + s := &S3Backend{ client: s3conn, bucket: bucket, logger: logger, + permitPool: NewPermitPool(maxParInt), } return s, nil } @@ -97,6 +111,9 @@ func newS3Backend(conf map[string]string, logger log.Logger) (Backend, error) { func (s *S3Backend) Put(entry *Entry) error { defer metrics.MeasureSince([]string{"s3", "put"}, time.Now()) + s.permitPool.Acquire() + defer s.permitPool.Release() + _, err := s.client.PutObject(&s3.PutObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(entry.Key), @@ -114,6 +131,9 @@ func (s *S3Backend) Put(entry *Entry) error { func (s *S3Backend) Get(key string) (*Entry, error) { defer metrics.MeasureSince([]string{"s3", "get"}, time.Now()) + s.permitPool.Acquire() + defer s.permitPool.Release() + resp, err := s.client.GetObject(&s3.GetObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), @@ -122,9 +142,8 @@ func (s *S3Backend) Get(key string) (*Entry, error) { // Return nil on 404s, error on anything else if awsErr.StatusCode() == 404 { return nil, nil - } else { - return nil, err } + return nil, err } if err != nil { return nil, err @@ -151,6 +170,9 @@ func (s *S3Backend) Get(key string) (*Entry, error) { func (s *S3Backend) Delete(key string) error { defer metrics.MeasureSince([]string{"s3", "delete"}, time.Now()) + s.permitPool.Acquire() + defer s.permitPool.Release() + _, err := s.client.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), @@ -168,6 +190,9 @@ func (s *S3Backend) Delete(key string) error { func (s *S3Backend) List(prefix string) ([]string, error) { defer metrics.MeasureSince([]string{"s3", "list"}, time.Now()) + s.permitPool.Acquire() + defer s.permitPool.Release() + params := &s3.ListObjectsV2Input{ Bucket: aws.String(s.bucket), Prefix: aws.String(prefix), From 5894434f7c8fce8dd2aec82b0468100300756d42 Mon Sep 17 00:00:00 2001 From: Dan Everton Date: Fri, 10 Mar 2017 13:19:26 +1000 Subject: [PATCH 2/4] Add missing imports. --- physical/s3.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/physical/s3.go b/physical/s3.go index 867a9b7c11e4..088ddb8dea42 100644 --- a/physical/s3.go +++ b/physical/s3.go @@ -6,6 +6,7 @@ import ( "io" "os" "sort" + "strconv" "strings" "time" @@ -16,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/errwrap" "github.com/hashicorp/vault/helper/awsutil" ) From 1590232464833986ee5657873c23590ee04c7fc5 Mon Sep 17 00:00:00 2001 From: Dan Everton Date: Fri, 10 Mar 2017 13:23:02 +1000 Subject: [PATCH 3/4] Update documentation for max_parallel for S3 --- website/source/docs/configuration/storage/s3.html.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/source/docs/configuration/storage/s3.html.md b/website/source/docs/configuration/storage/s3.html.md index cbd0acff658b..e7472f789f91 100644 --- a/website/source/docs/configuration/storage/s3.html.md +++ b/website/source/docs/configuration/storage/s3.html.md @@ -55,6 +55,9 @@ cause Vault to attempt to retrieve credentials from the AWS metadata service. - `session_token` `(string: "")` – Specifies the AWS session token. This can also be provided via the environment variable `AWS_SESSION_TOKEN`. +- `max_parallel` `(int: 128)` – Specifies The maximum number of concurrent + requests to S3. + ## `s3` Examples ### Default Example From c9314db9a9a8ca8c900888090c7034eb91783cef Mon Sep 17 00:00:00 2001 From: Dan Everton Date: Fri, 10 Mar 2017 13:58:18 +1000 Subject: [PATCH 4/4] The max_parallel parameter is actually a string. --- website/source/docs/configuration/storage/azure.html.md | 2 +- website/source/docs/configuration/storage/consul.html.md | 2 +- website/source/docs/configuration/storage/dynamodb.html.md | 2 +- website/source/docs/configuration/storage/google-cloud.html.md | 2 +- website/source/docs/configuration/storage/s3.html.md | 2 +- website/source/docs/configuration/storage/swift.html.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/website/source/docs/configuration/storage/azure.html.md b/website/source/docs/configuration/storage/azure.html.md index 4bb9468066e1..f8a272048ab3 100644 --- a/website/source/docs/configuration/storage/azure.html.md +++ b/website/source/docs/configuration/storage/azure.html.md @@ -43,7 +43,7 @@ The current implementation is limited to a maximum of 4 megabytes per blob. - `container` `(string: )` – Specifies the Azure Storage Blob container name. -- `max_parallel` `(int: 128)` – Specifies The maximum number of concurrent +- `max_parallel` `(string: "128")` – Specifies The maximum number of concurrent requests to Azure. ## `azure` Examples diff --git a/website/source/docs/configuration/storage/consul.html.md b/website/source/docs/configuration/storage/consul.html.md index edd4f5f8d596..981334d182f8 100644 --- a/website/source/docs/configuration/storage/consul.html.md +++ b/website/source/docs/configuration/storage/consul.html.md @@ -69,7 +69,7 @@ at Consul's service discovery layer. - `disable_registration` `(bool: false)` – Specifies whether Vault should register itself with Consul. -- `max_parallel` `(int: 128)` – Specifies the maximum number of concurrent +- `max_parallel` `(string: "128")` – Specifies the maximum number of concurrent requests to Consul. - `path` `(string: "vault/")` – Specifies the path in Consul's key-value store diff --git a/website/source/docs/configuration/storage/dynamodb.html.md b/website/source/docs/configuration/storage/dynamodb.html.md index 3dc21fb57d02..2bcbd077ca97 100644 --- a/website/source/docs/configuration/storage/dynamodb.html.md +++ b/website/source/docs/configuration/storage/dynamodb.html.md @@ -43,7 +43,7 @@ see the [official AWS DynamoDB documentation][dynamodb-rw-capacity]. to run Vault in high availability mode. This can also be provided via the environment variable `DYNAMODB_HA_ENABLED`. -- `max_parallel` `(int: 128)` – Specifies the maximum number of concurrent +- `max_parallel` `(string: "128")` – Specifies the maximum number of concurrent requests. - `region` `(string "us-east-1")` – Specifies the AWS region. This can also be diff --git a/website/source/docs/configuration/storage/google-cloud.html.md b/website/source/docs/configuration/storage/google-cloud.html.md index e19f1c2f1c7c..15e51af498ab 100644 --- a/website/source/docs/configuration/storage/google-cloud.html.md +++ b/website/source/docs/configuration/storage/google-cloud.html.md @@ -39,7 +39,7 @@ storage "gcs" { in [JSON format][gcs-private-key]. This can also be provided via the environment variable `GOOGLE_APPLICATION_CREDENTIALS`. -- `max_parallel` `(int: 128)` – Specifies the maximum number of concurrent +- `max_parallel` `(string: "128")` – Specifies the maximum number of concurrent requests. ## `gcs` Examples diff --git a/website/source/docs/configuration/storage/s3.html.md b/website/source/docs/configuration/storage/s3.html.md index e7472f789f91..a020c660da75 100644 --- a/website/source/docs/configuration/storage/s3.html.md +++ b/website/source/docs/configuration/storage/s3.html.md @@ -55,7 +55,7 @@ cause Vault to attempt to retrieve credentials from the AWS metadata service. - `session_token` `(string: "")` – Specifies the AWS session token. This can also be provided via the environment variable `AWS_SESSION_TOKEN`. -- `max_parallel` `(int: 128)` – Specifies The maximum number of concurrent +- `max_parallel` `(string: "128")` – Specifies The maximum number of concurrent requests to S3. ## `s3` Examples diff --git a/website/source/docs/configuration/storage/swift.html.md b/website/source/docs/configuration/storage/swift.html.md index a78678ab8cd9..5848bf0025df 100644 --- a/website/source/docs/configuration/storage/swift.html.md +++ b/website/source/docs/configuration/storage/swift.html.md @@ -40,7 +40,7 @@ storage "swift" { container. This can also be provided via the environment variable `OS_CONTAINER`. -- `max_parallel` `(int: 128)` – The maximum number of concurrent requests. +- `max_parallel` `(string: "128")` – The maximum number of concurrent requests. - `password` `(string: )` – Specifies the OpenStack password. This can also be provided via the environment variable `OS_PASSWORD`.