From 9f9327c20aab24d677bce23f140bee371bdf80c6 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Fri, 7 Jul 2017 08:56:03 -0400 Subject: [PATCH 1/4] Use RemoteCredProvider instead of EC2RoleProvider to better handle ECS and other cases --- builtin/logical/aws/client.go | 6 +++++- helper/awsutil/generate_credentials.go | 20 +++++++++----------- physical/s3.go | 7 +++++-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/builtin/logical/aws/client.go b/builtin/logical/aws/client.go index 545c6856aafe..a9b1e78d5980 100644 --- a/builtin/logical/aws/client.go +++ b/builtin/logical/aws/client.go @@ -2,6 +2,7 @@ package aws import ( "fmt" + "os" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -31,7 +32,10 @@ func getRootConfig(s logical.Storage) (*aws.Config, error) { } if credsConfig.Region == "" { - credsConfig.Region = "us-east-1" + credsConfig.Region = os.Getenv("AWS_REGION") + if credsConfig.Region == "" { + credsConfig.Region = "us-east-1" + } } credsConfig.HTTPClient = cleanhttp.DefaultClient() diff --git a/helper/awsutil/generate_credentials.go b/helper/awsutil/generate_credentials.go index 7399a5ca367c..d80b98183b04 100644 --- a/helper/awsutil/generate_credentials.go +++ b/helper/awsutil/generate_credentials.go @@ -6,9 +6,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/aws/defaults" ) type CredentialsConfig struct { @@ -65,14 +63,14 @@ func (c *CredentialsConfig) GenerateCredentialChain() (*credentials.Credentials, Profile: c.Profile, }) - // Add the instance metadata role provider - providers = append(providers, &ec2rolecreds.EC2RoleProvider{ - Client: ec2metadata.New(session.New(&aws.Config{ - Region: aws.String(c.Region), - HTTPClient: c.HTTPClient, - })), - ExpiryWindow: 15, - }) + // Add the remote provider + def := defaults.Get() + if c.Region != "" { + def.Config.Region = aws.String(c.Region) + } + def.Config.HTTPClient = c.HTTPClient + + providers = append(providers, defaults.RemoteCredProvider(def.Config, def.Handlers)) // Create the credentials required to access the API. creds := credentials.NewChainCredentials(providers) diff --git a/physical/s3.go b/physical/s3.go index 9061f7fd98a3..ab930904eca8 100644 --- a/physical/s3.go +++ b/physical/s3.go @@ -64,9 +64,12 @@ func newS3Backend(conf map[string]string, logger log.Logger) (Backend, error) { } region := os.Getenv("AWS_DEFAULT_REGION") if region == "" { - region = conf["region"] + region = os.Getenv("AWS_REGION") if region == "" { - region = "us-east-1" + region = conf["region"] + if region == "" { + region = "us-east-1" + } } } From 2bd12532376ca90c8251675b5ddebfa7c9eecfbf Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Fri, 7 Jul 2017 12:46:51 -0400 Subject: [PATCH 2/4] Address feedback --- helper/awsutil/generate_credentials.go | 2 +- physical/dynamodb.go | 43 ++++++++++++++++---------- physical/s3.go | 4 +-- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/helper/awsutil/generate_credentials.go b/helper/awsutil/generate_credentials.go index d80b98183b04..218191dfbcc2 100644 --- a/helper/awsutil/generate_credentials.go +++ b/helper/awsutil/generate_credentials.go @@ -70,7 +70,7 @@ func (c *CredentialsConfig) GenerateCredentialChain() (*credentials.Credentials, } def.Config.HTTPClient = c.HTTPClient - providers = append(providers, defaults.RemoteCredProvider(def.Config, def.Handlers)) + providers = append(providers, defaults.RemoteCredProvider(*def.Config, def.Handlers)) // Create the credentials required to access the API. creds := credentials.NewChainCredentials(providers) diff --git a/physical/dynamodb.go b/physical/dynamodb.go index c9eded4ede9a..9bfdc6123bd9 100644 --- a/physical/dynamodb.go +++ b/physical/dynamodb.go @@ -3,6 +3,7 @@ package physical import ( "fmt" "math" + "net/http" "os" pkgPath "path" "sort" @@ -16,14 +17,14 @@ import ( "github.com/armon/go-metrics" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" "github.com/hashicorp/errwrap" + cleanhttp "github.com/hashicorp/go-cleanhttp" "github.com/hashicorp/go-uuid" + "github.com/hashicorp/vault/helper/awsutil" + "github.com/hashicorp/vault/helper/consts" ) const ( @@ -166,29 +167,37 @@ func newDynamoDBBackend(conf map[string]string, logger log.Logger) (Backend, err if endpoint == "" { endpoint = conf["endpoint"] } - region := os.Getenv("AWS_DEFAULT_REGION") + region := os.Getenv("AWS_REGION") if region == "" { - region = conf["region"] + region = os.Getenv("AWS_DEFAULT_REGION") if region == "" { - region = DefaultDynamoDBRegion + region = conf["region"] + if region == "" { + region = DefaultDynamoDBRegion + } } } - creds := credentials.NewChainCredentials([]credentials.Provider{ - &credentials.StaticProvider{Value: credentials.Value{ - AccessKeyID: accessKey, - SecretAccessKey: secretKey, - SessionToken: sessionToken, - }}, - &credentials.EnvProvider{}, - &credentials.SharedCredentialsProvider{Filename: "", Profile: ""}, - &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())}, - }) + credsConfig := &awsutil.CredentialsConfig{ + AccessKey: accessKey, + SecretKey: secretKey, + SessionToken: sessionToken, + } + creds, err := credsConfig.GenerateCredentialChain() + if err != nil { + return nil, err + } + + pooledTransport := cleanhttp.DefaultPooledTransport() + pooledTransport.MaxIdleConnsPerHost = consts.ExpirationRestoreWorkerCount awsConf := aws.NewConfig(). WithCredentials(creds). WithRegion(region). - WithEndpoint(endpoint) + WithEndpoint(endpoint). + WithHTTPClient(&http.Client{ + Transport: pooledTransport, + }) client := dynamodb.New(session.New(awsConf)) if err := ensureTableExists(client, table, readCapacity, writeCapacity); err != nil { diff --git a/physical/s3.go b/physical/s3.go index ab930904eca8..13df06bc4fca 100644 --- a/physical/s3.go +++ b/physical/s3.go @@ -62,9 +62,9 @@ func newS3Backend(conf map[string]string, logger log.Logger) (Backend, error) { if endpoint == "" { endpoint = conf["endpoint"] } - region := os.Getenv("AWS_DEFAULT_REGION") + region := os.Getenv("AWS_REGION") if region == "" { - region = os.Getenv("AWS_REGION") + region = os.Getenv("AWS_DEFAULT_REGION") if region == "" { region = conf["region"] if region == "" { From 363458a92a964b4ae9a635411467973773c609f9 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Fri, 7 Jul 2017 18:04:24 -0400 Subject: [PATCH 3/4] Update from feedback --- builtin/logical/aws/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/builtin/logical/aws/client.go b/builtin/logical/aws/client.go index a9b1e78d5980..6415c144186c 100644 --- a/builtin/logical/aws/client.go +++ b/builtin/logical/aws/client.go @@ -34,7 +34,10 @@ func getRootConfig(s logical.Storage) (*aws.Config, error) { if credsConfig.Region == "" { credsConfig.Region = os.Getenv("AWS_REGION") if credsConfig.Region == "" { - credsConfig.Region = "us-east-1" + credsConfig.Region = os.Getenv("AWS_DEFAULT_REGION") + if credsConfig.Region == "" { + credsConfig.Region = "us-east-1" + } } } From f9ab4219a204af14dc9afd428557df69622bc113 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Wed, 12 Jul 2017 14:59:22 -0400 Subject: [PATCH 4/4] Address feedback --- builtin/logical/aws/path_config_root.go | 3 --- website/source/api/secret/aws/index.html.md | 12 +++++++++--- website/source/docs/auth/aws.html.md | 8 ++++---- website/source/docs/configuration/storage/s3.html.md | 5 +++-- website/source/docs/secrets/aws/index.html.md | 12 +++++++++--- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/builtin/logical/aws/path_config_root.go b/builtin/logical/aws/path_config_root.go index 0d1d1d5906fe..754e5b2a4318 100644 --- a/builtin/logical/aws/path_config_root.go +++ b/builtin/logical/aws/path_config_root.go @@ -37,9 +37,6 @@ func pathConfigRoot() *framework.Path { func pathConfigRootWrite( req *logical.Request, data *framework.FieldData) (*logical.Response, error) { region := data.Get("region").(string) - if region == "" { - region = "us-east-1" - } entry, err := logical.StorageEntryJSON("config/root", rootConfig{ AccessKey: data.Get("access_key").(string), diff --git a/website/source/api/secret/aws/index.html.md b/website/source/api/secret/aws/index.html.md index 25dc2682998c..5b58cecf454d 100644 --- a/website/source/api/secret/aws/index.html.md +++ b/website/source/api/secret/aws/index.html.md @@ -23,13 +23,17 @@ are multiple ways to pass root IAM credentials to the Vault server, specified below with the highest precedence first. If credentials already exist, this will overwrite them. +The official AWS SDK is used for sourcing credentials from env vars, shared +files, or IAM/ECS instances. + - Static credentials provided to the API as a payload - Credentials in the `AWS_ACCESS_KEY`, `AWS_SECRET_KEY`, and `AWS_REGION` environment variables **on the server** -- Querying the EC2 metadata service if the **Vault server** is on EC2 and has - querying capabilities +- Shared credentials files + +- Assigned IAM role or ECS task role credentials At present, this endpoint does not confirm that the provided AWS credentials are valid AWS credentials with proper permissions. @@ -44,7 +48,9 @@ valid AWS credentials with proper permissions. - `secret_key` `(string: )` – Specifies the AWS secret access key. -- `region` `(string: )` – Specifies the AWS region. +- `region` `(string: )` – Specifies the AWS region. If not set it + will use the `AWS_REGION` env var, `AWS_DEFAULT_REGION` env var, or + `us-east-1` in that order. ### Sample Payload diff --git a/website/source/docs/auth/aws.html.md b/website/source/docs/auth/aws.html.md index 1c7e260009e4..3a5efd077d96 100644 --- a/website/source/docs/auth/aws.html.md +++ b/website/source/docs/auth/aws.html.md @@ -546,10 +546,10 @@ $ vault auth -method=aws header_value=vault.example.com role=dev-role-iam This assumes you have AWS credentials configured in the standard locations AWS SDKs search for credentials (environment variables, ~/.aws/credentials, IAM -instance profile in that order). If you do not have IAM credentials available at -any of these locations, you can explicitly pass them in on the command line -(though this is not recommended), omitting `aws_security_token` if not -applicable . +instance profile, or ECS task role, in that order). If you do not have IAM +credentials available at any of these locations, you can explicitly pass them +in on the command line (though this is not recommended), omitting +`aws_security_token` if not applicable. ``` $ vault auth -method=aws header_value=vault.example.com role=dev-role-iam \ diff --git a/website/source/docs/configuration/storage/s3.html.md b/website/source/docs/configuration/storage/s3.html.md index 1cecd30e8d9a..7b167bd114e3 100644 --- a/website/source/docs/configuration/storage/s3.html.md +++ b/website/source/docs/configuration/storage/s3.html.md @@ -35,10 +35,11 @@ storage "s3" { - `endpoint` `(string: "")` – Specifies an alternative, AWS compatible, S3 endpoint. This can also be provided via the environment variable - `AWS_DEFAULT_REGION`. + `AWS_S3_ENDPOINT`. - `region` `(string "us-east-1")` – Specifies the AWS region. This can also be - provided via the environment variable `AWS_DEFAULT_REGION`. + provided via the environment variable `AWS_REGION` or `AWS_DEFAULT_REGION`, + in that order of preference. The following settings are used for authenticating to AWS. If you are running your Vault server on an EC2 instance, you can also make use of the EC2 diff --git a/website/source/docs/secrets/aws/index.html.md b/website/source/docs/secrets/aws/index.html.md index a062bce03e6d..f15806b1c19f 100644 --- a/website/source/docs/secrets/aws/index.html.md +++ b/website/source/docs/secrets/aws/index.html.md @@ -43,10 +43,16 @@ The following parameters are required: credentials. - `secret_key` - the AWS secret key that has permission to manage IAM credentials. -- `region` the AWS region for API calls. -Note: the client uses the official AWS SDK and will use environment variable or IAM -role-provided credentials if available. +The following parameter is optional: + +- `region` the AWS region for API calls. If not provided, the `AWS_REGION` and + `AWS_DEFAULT_REGION` env vars will be used, in that order. If there is still + no region, `us-east-1` will be used as a fallback. + +Note: the client uses the official AWS SDK and will use the specified +credentials, environment credentials, shared file credentials, or IAM role/ECS +task credentials in that order. The next step is to configure a role. A role is a logical name that maps to a policy used to generated those credentials.