Skip to content

Commit

Permalink
Adds logging to awsutil GenerateCredentialChain()
Browse files Browse the repository at this point in the history 
Adds debug and warn logging around AWS credential chain generation,
specifically to help users debugging auto-unseal problems on AWS, by
logging which role is being used in the case of a webidentity token.

Adds a deferred call to flush the log output as well, to ensure logs
are output in the event of an initialization failure.
  • Loading branch information
@tvoran
tvoran committed Aug 21, 2020
1 parent d2aa6c7 commit 9162d73
Show file tree
Hide file tree
Showing 121 changed files with 20,265 additions and 1,193 deletions.
5 changes: 3 additions & 2 deletions builtin/credential/aws/cli.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func (h *CLIHandler) Auth(c *api.Client, m map[string]string) (*api.Secret, erro
headerValue = ""
}

creds, err := RetrieveCreds(m["aws_access_key_id"], m["aws_secret_access_key"], m["aws_security_token"])
creds, err := RetrieveCreds(m["aws_access_key_id"], m["aws_secret_access_key"], m["aws_security_token"], hclog.Default())
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -128,11 +128,12 @@ func (h *CLIHandler) Auth(c *api.Client, m map[string]string) (*api.Secret, erro
return secret, nil
}

func RetrieveCreds(accessKey, secretKey, sessionToken string) (*credentials.Credentials, error) {
func RetrieveCreds(accessKey, secretKey, sessionToken string, logger hclog.Logger) (*credentials.Credentials, error) {
credConfig := &awsutil.CredentialsConfig{
AccessKey: accessKey,
SecretKey: secretKey,
SessionToken: sessionToken,
Logger: logger,
}
creds, err := credConfig.GenerateCredentialChain()
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions builtin/credential/aws/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, region, clientType string) (*aws.Config, error) {
credsConfig := &awsutil.CredentialsConfig{
Region: region,
Logger: b.Logger(),
}

// Read the configured secret key and access key
Expand Down
5 changes: 4 additions & 1 deletion builtin/credential/aws/path_role_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ import (
"testing"

"github.com/go-test/deep"
log "github.com/hashicorp/go-hclog"
vlttesting "github.com/hashicorp/vault/helper/testhelpers/logical"
"github.com/hashicorp/vault/sdk/helper/awsutil"
"github.com/hashicorp/vault/sdk/helper/logging"
"github.com/hashicorp/vault/sdk/helper/policyutil"
"github.com/hashicorp/vault/sdk/helper/strutil"
"github.com/hashicorp/vault/sdk/logical"
Expand Down Expand Up @@ -1009,7 +1011,8 @@ func TestRoleResolutionWithSTSEndpointConfigured(t *testing.T) {
}

// Ensure aws credentials are available locally for testing.
credsConfig := &awsutil.CredentialsConfig{}
logger := logging.NewVaultLogger(log.Debug)
credsConfig := &awsutil.CredentialsConfig{Logger: logger}
credsChain, err := credsConfig.GenerateCredentialChain()
if err != nil {
t.Fatal(err)
Expand Down
4 changes: 2 additions & 2 deletions builtin/logical/aws/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func (b *backend) clientIAM(ctx context.Context, s logical.Storage) (iamiface.IA
return b.iamClient, nil
}

iamClient, err := nonCachedClientIAM(ctx, s)
iamClient, err := nonCachedClientIAM(ctx, s, b.Logger())
if err != nil {
return nil, err
}
Expand All @@ -148,7 +148,7 @@ func (b *backend) clientSTS(ctx context.Context, s logical.Storage) (stsiface.ST
return b.stsClient, nil
}

stsClient, err := nonCachedClientSTS(ctx, s)
stsClient, err := nonCachedClientSTS(ctx, s, b.Logger())
if err != nil {
return nil, err
}
Expand Down
13 changes: 8 additions & 5 deletions builtin/logical/aws/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,13 @@ import (
"github.com/aws/aws-sdk-go/service/sts"
"github.com/hashicorp/errwrap"
cleanhttp "github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/sdk/helper/awsutil"
"github.com/hashicorp/vault/sdk/logical"
)

// NOTE: The caller is required to ensure that b.clientMutex is at least read locked
func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*aws.Config, error) {
func getRootConfig(ctx context.Context, s logical.Storage, clientType string, logger hclog.Logger) (*aws.Config, error) {
credsConfig := &awsutil.CredentialsConfig{}
var endpoint string
var maxRetries int = aws.UseServiceDefaultRetries
Expand Down Expand Up @@ -55,6 +56,8 @@ func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*

credsConfig.HTTPClient = cleanhttp.DefaultClient()

credsConfig.Logger = logger

creds, err := credsConfig.GenerateCredentialChain()
if err != nil {
return nil, err
Expand All @@ -69,8 +72,8 @@ func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*
}, nil
}

func nonCachedClientIAM(ctx context.Context, s logical.Storage) (*iam.IAM, error) {
awsConfig, err := getRootConfig(ctx, s, "iam")
func nonCachedClientIAM(ctx context.Context, s logical.Storage, logger hclog.Logger) (*iam.IAM, error) {
awsConfig, err := getRootConfig(ctx, s, "iam", logger)
if err != nil {
return nil, err
}
Expand All @@ -85,8 +88,8 @@ func nonCachedClientIAM(ctx context.Context, s logical.Storage) (*iam.IAM, error
return client, nil
}

func nonCachedClientSTS(ctx context.Context, s logical.Storage) (*sts.STS, error) {
awsConfig, err := getRootConfig(ctx, s, "sts")
func nonCachedClientSTS(ctx context.Context, s logical.Storage, logger hclog.Logger) (*sts.STS, error) {
awsConfig, err := getRootConfig(ctx, s, "sts", logger)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions command/agent/auth/aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func NewAWSAuthMethod(conf *auth.AuthConfig) (auth.AuthMethod, error) {

// Do an initial population of the creds because we want to err right away if we can't
// even get a first set.
creds, err := awsauth.RetrieveCreds(accessKey, secretKey, sessionToken)
creds, err := awsauth.RetrieveCreds(accessKey, secretKey, sessionToken, a.logger)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -272,7 +272,7 @@ func (a *awsMethod) checkCreds(accessKey, secretKey, sessionToken string) error
defer a.credLock.Unlock()

a.logger.Trace("checking for new credentials")
currentCreds, err := awsauth.RetrieveCreds(accessKey, secretKey, sessionToken)
currentCreds, err := awsauth.RetrieveCreds(accessKey, secretKey, sessionToken, a.logger)
if err != nil {
return err
}
Expand Down
28 changes: 16 additions & 12 deletions command/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,12 @@ func (c *ServerCommand) AutocompleteFlags() complete.Flags {
return c.Flags().Completions()
}

func (c *ServerCommand) flushLog() {
c.logger.(hclog.OutputResettable).ResetOutputWithFlush(&hclog.LoggerOptions{
Output: c.logOutput,
}, c.gatedWriter)
}

func (c *ServerCommand) parseConfig() (*server.Config, error) {
// Load the configuration
var config *server.Config
Expand Down Expand Up @@ -427,6 +433,9 @@ func (c *ServerCommand) runRecoveryMode() int {
JSONFormat: logFormat == logging.JSONFormat,
})

// Ensure logging is flushed if initialization fails
defer c.flushLog()

logLevelStr, err := c.adjustLogLevel(config, logLevelWasNotSet)
if err != nil {
c.UI.Error(err.Error())
Expand Down Expand Up @@ -669,9 +678,7 @@ func (c *ServerCommand) runRecoveryMode() int {
c.UI.Output("==> Vault server started! Log data will stream in below:\n")
}

c.logger.(hclog.OutputResettable).ResetOutputWithFlush(&hclog.LoggerOptions{
Output: c.logOutput,
}, c.gatedWriter)
c.flushLog()

for {
select {
Expand Down Expand Up @@ -900,6 +907,9 @@ func (c *ServerCommand) Run(args []string) int {
})
}

// Ensure logging is flushed if initialization fails
defer c.flushLog()

allLoggers := []log.Logger{c.logger}

logLevelStr, err := c.adjustLogLevel(config, logLevelWasNotSet)
Expand Down Expand Up @@ -1778,9 +1788,7 @@ CLUSTER_SYNTHESIS_COMPLETE:
}

// Release the log gate.
c.logger.(hclog.OutputResettable).ResetOutputWithFlush(&hclog.LoggerOptions{
Output: c.logOutput,
}, c.gatedWriter)
c.flushLog()

// Write out the PID to the file now that server has successfully started
if err := c.storePidFile(config.PidFile); err != nil {
Expand Down Expand Up @@ -2186,9 +2194,7 @@ func (c *ServerCommand) enableThreeNodeDevCluster(base *vault.CoreConfig, info m
}

// Release the log gate.
c.logger.(hclog.OutputResettable).ResetOutputWithFlush(&hclog.LoggerOptions{
Output: c.logOutput,
}, c.gatedWriter)
c.flushLog()

// Wait for shutdown
shutdownTriggered := false
Expand Down Expand Up @@ -2423,9 +2429,7 @@ func (c *ServerCommand) storageMigrationActive(backend physical.Backend) bool {
c.UI.Warn("\nWARNING! Unable to read storage migration status.")

// unexpected state, so stop buffering log messages
c.logger.(hclog.OutputResettable).ResetOutputWithFlush(&hclog.LoggerOptions{
Output: c.logOutput,
}, c.gatedWriter)
c.flushLog()
}
c.logger.Warn("storage migration check error", "error", err.Error())

Expand Down
10 changes: 5 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ require (
github.com/hashicorp/go-cleanhttp v0.5.1
github.com/hashicorp/go-gcp-common v0.6.0
github.com/hashicorp/go-hclog v0.14.1
github.com/hashicorp/go-kms-wrapping v0.5.12
github.com/hashicorp/go-kms-wrapping v0.5.15-0.20200820184447-e735e02841a5
github.com/hashicorp/go-memdb v1.0.2
github.com/hashicorp/go-msgpack v0.5.5
github.com/hashicorp/go-multierror v1.1.0
Expand Down Expand Up @@ -90,8 +90,8 @@ require (
github.com/hashicorp/vault-plugin-secrets-kv v0.5.6
github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.1.2
github.com/hashicorp/vault-plugin-secrets-openldap v0.1.5
github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086
github.com/hashicorp/vault/sdk v0.1.14-0.20200717191844-f687267c8086
github.com/hashicorp/vault/api v1.0.5-0.20200805123347-1ef507638af6
github.com/hashicorp/vault/sdk v0.1.14-0.20200805123347-1ef507638af6
github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4
github.com/jcmturner/gokrb5/v8 v8.0.0
github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f
Expand Down Expand Up @@ -133,7 +133,7 @@ require (
github.com/sasha-s/go-deadlock v0.2.0
github.com/sethvargo/go-limiter v0.3.0
github.com/shirou/gopsutil v2.20.6-0.20200630091542-01afd763e6c0+incompatible
github.com/stretchr/testify v1.5.1
github.com/stretchr/testify v1.6.1
github.com/tidwall/pretty v1.0.1 // indirect
github.com/ulikunitz/xz v0.5.7 // indirect
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
Expand All @@ -150,7 +150,7 @@ require (
golang.org/x/tools v0.0.0-20200521155704-91d71f6c2f04
google.golang.org/api v0.29.0
google.golang.org/grpc v1.29.1
google.golang.org/protobuf v1.24.0
google.golang.org/protobuf v1.25.0
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce
gopkg.in/ory-am/dockertest.v3 v3.3.4
gopkg.in/square/go-jose.v2 v2.5.1
Expand Down
Loading

0 comments on commit 9162d73

Please sign in to comment.