From c5948734c42e359a76346abd056ee0010298ef9a Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Fri, 22 Apr 2022 13:44:15 -0400 Subject: [PATCH 01/11] Add initial_backoff to auto-auth method --- command/agent/config/config.go | 26 +++++++---- command/agent/config/config_test.go | 43 +++++++++++++++++++ .../config-method-initial-backoff.hcl | 20 +++++++++ 3 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 command/agent/config/test-fixtures/config-method-initial-backoff.hcl diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 7a105573da4f..502b8d67f4dc 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -108,14 +108,16 @@ type AutoAuth struct { // Method represents the configuration for the authentication backend type Method struct { - Type string - MountPath string `hcl:"mount_path"` - WrapTTLRaw interface{} `hcl:"wrap_ttl"` - WrapTTL time.Duration `hcl:"-"` - MaxBackoffRaw interface{} `hcl:"max_backoff"` - MaxBackoff time.Duration `hcl:"-"` - Namespace string `hcl:"namespace"` - Config map[string]interface{} + Type string + MountPath string `hcl:"mount_path"` + WrapTTLRaw interface{} `hcl:"wrap_ttl"` + WrapTTL time.Duration `hcl:"-"` + InitialBackoffRaw interface{} `hcl:"initial_backoff"` + InitialBackoff time.Duration `hcl:"-"` + MaxBackoffRaw interface{} `hcl:"max_backoff"` + MaxBackoff time.Duration `hcl:"-"` + Namespace string `hcl:"namespace"` + Config map[string]interface{} } // Sink defines a location to write the authenticated token @@ -470,6 +472,14 @@ func parseAutoAuth(result *Config, list *ast.ObjectList) error { result.AutoAuth.Method.MaxBackoffRaw = nil } + if result.AutoAuth.Method.InitialBackoffRaw != nil { + var err error + if result.AutoAuth.Method.InitialBackoff, err = parseutil.ParseDurationSecond(result.AutoAuth.Method.InitialBackoffRaw); err != nil { + return err + } + result.AutoAuth.Method.InitialBackoffRaw = nil + } + return nil } diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index 2bb8844bac15..88569af72778 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -290,6 +290,49 @@ func TestLoadConfigFile_Method_Wrapping(t *testing.T) { } } +func TestLoadConfigFile_Method_InitialBackoff(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-method-initial-backoff.hcl") + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + WrapTTL: 5 * time.Minute, + InitialBackoff: 5 * time.Second, + MaxBackoff: 2 * time.Minute, + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Retry: &Retry{ + NumRetries: 12, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + func TestLoadConfigFile_AgentCache_NoAutoAuth(t *testing.T) { config, err := LoadConfig("./test-fixtures/config-cache-no-auto_auth.hcl") if err != nil { diff --git a/command/agent/config/test-fixtures/config-method-initial-backoff.hcl b/command/agent/config/test-fixtures/config-method-initial-backoff.hcl new file mode 100644 index 000000000000..d70959822dd0 --- /dev/null +++ b/command/agent/config/test-fixtures/config-method-initial-backoff.hcl @@ -0,0 +1,20 @@ +pid_file = "./pidfile" + +auto_auth { + method { + type = "aws" + wrap_ttl = 300 + config = { + role = "foobar" + } + max_backoff = "2m" + initial_backoff = "5s" + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + } +} From 36a1be5d952c9fb3979cdd30265e0990d99d9603 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Wed, 27 Apr 2022 11:26:30 -0400 Subject: [PATCH 02/11] Disable retries in client --- command/agent.go | 10 +++++- command/agent/auth/auth.go | 44 +++++++++++++++++++----- command/agent/auth/auth_test.go | 59 ++++++++++++++++++++++++++++++--- 3 files changed, 100 insertions(+), 13 deletions(-) diff --git a/command/agent.go b/command/agent.go index 6bafd4cb072d..93fffbf28cc4 100644 --- a/command/agent.go +++ b/command/agent.go @@ -790,9 +790,17 @@ func (c *AgentCommand) Run(args []string) int { // Start auto-auth and sink servers if method != nil { enableTokenCh := len(config.Templates) > 0 + + // Auth Handler is going to set its own retry values, so we want to + // work on a copy of the client to not affect other subsystems. + clonedClient, err := c.client.Clone() + if err != nil { + c.UI.Error(fmt.Sprintf("Error cloning client for auth handler: %v", err)) + return 1 + } ah := auth.NewAuthHandler(&auth.AuthHandlerConfig{ Logger: c.logger.Named("auth.handler"), - Client: c.client, + Client: clonedClient, WrapTTL: config.AutoAuth.Method.WrapTTL, MaxBackoff: config.AutoAuth.Method.MaxBackoff, EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials, diff --git a/command/agent/auth/auth.go b/command/agent/auth/auth.go index c00028608803..c30391d83344 100644 --- a/command/agent/auth/auth.go +++ b/command/agent/auth/auth.go @@ -15,8 +15,8 @@ import ( ) const ( - initialBackoff = 1 * time.Second - defaultMaxBackoff = 5 * time.Minute + defaultInitialBackoff = 1 * time.Second + defaultMaxBackoff = 5 * time.Minute ) // AuthMethod is the interface that auto-auth methods implement for the agent @@ -55,6 +55,7 @@ type AuthHandler struct { random *rand.Rand wrapTTL time.Duration maxBackoff time.Duration + initialBackoff time.Duration enableReauthOnNewCredentials bool enableTemplateTokenCh bool } @@ -64,12 +65,18 @@ type AuthHandlerConfig struct { Client *api.Client WrapTTL time.Duration MaxBackoff time.Duration + InitialBackoff time.Duration Token string EnableReauthOnNewCredentials bool EnableTemplateTokenCh bool } func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { + initial := conf.InitialBackoff + if initial <= 0 { + initial = defaultInitialBackoff + } + ah := &AuthHandler{ // This is buffered so that if we try to output after the sink server // has been shut down, during agent shutdown, we won't block @@ -80,6 +87,7 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { client: conf.Client, random: rand.New(rand.NewSource(int64(time.Now().Nanosecond()))), wrapTTL: conf.WrapTTL, + initialBackoff: initial, maxBackoff: conf.MaxBackoff, enableReauthOnNewCredentials: conf.EnableReauthOnNewCredentials, enableTemplateTokenCh: conf.EnableTemplateTokenCh, @@ -104,7 +112,15 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { return errors.New("auth handler: nil auth method") } - backoff := newAgentBackoff(ah.maxBackoff) + if ah.initialBackoff <= 0 { + ah.initialBackoff = defaultInitialBackoff + } + + backoff := newAgentBackoff(ah.initialBackoff, ah.maxBackoff) + + if backoff.current >= backoff.max { + return errors.New("auth handler: initial backoff cannot be greater than max backoff") + } ah.logger.Info("starting auth handler") defer func() { @@ -164,6 +180,10 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { clientToUse = ah.client } + // Disable retry on the client to ensure our backoffOrQuit function is + // the only source of retry/backoff. + clientToUse.SetMaxRetries(0) + var secret *api.Secret = new(api.Secret) if first && ah.token != "" { ah.logger.Debug("using preloaded token") @@ -258,7 +278,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } am.CredSuccess() - backoff.reset() + backoff.reset(ah.initialBackoff) select { case <-ctx.Done(): @@ -290,7 +310,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } am.CredSuccess() - backoff.reset() + backoff.reset(ah.initialBackoff) } if watcher != nil { @@ -346,14 +366,18 @@ type agentBackoff struct { current time.Duration } -func newAgentBackoff(max time.Duration) *agentBackoff { +func newAgentBackoff(initial, max time.Duration) *agentBackoff { if max <= 0 { max = defaultMaxBackoff } + if initial <= 0 { + initial = defaultInitialBackoff + } + return &agentBackoff{ max: max, - current: initialBackoff, + current: initial, } } @@ -371,7 +395,11 @@ func (b *agentBackoff) next() { b.current = maxBackoff - time.Duration(trim) } -func (b *agentBackoff) reset() { +func (b *agentBackoff) reset(initialBackoff time.Duration) { + if initialBackoff <= 0 { + initialBackoff = defaultInitialBackoff + } + b.current = initialBackoff } diff --git a/command/agent/auth/auth_test.go b/command/agent/auth/auth_test.go index 05c24fe1fa39..925a8b05f9bf 100644 --- a/command/agent/auth/auth_test.go +++ b/command/agent/auth/auth_test.go @@ -109,10 +109,10 @@ consumption: func TestAgentBackoff(t *testing.T) { max := 1024 * time.Second - backoff := newAgentBackoff(max) + backoff := newAgentBackoff(defaultInitialBackoff, max) // Test initial value - if backoff.current != initialBackoff { + if backoff.current != defaultInitialBackoff { t.Fatalf("expected 1s initial backoff, got: %v", backoff.current) } @@ -138,8 +138,59 @@ func TestAgentBackoff(t *testing.T) { } // Test reset - backoff.reset() - if backoff.current != initialBackoff { + backoff.reset(defaultInitialBackoff) + if backoff.current != defaultInitialBackoff { t.Fatalf("expected 1s backoff after reset, got: %v", backoff.current) } } + +func TestAgentInitialBackoffCustom(t *testing.T) { + type test struct { + initialBackoff time.Duration + want time.Duration + } + + tests := []test{ + {initialBackoff: 0 * time.Second, want: 1 * time.Second}, + {initialBackoff: 1 * time.Second, want: 1 * time.Second}, + {initialBackoff: 5 * time.Second, want: 5 * time.Second}, + {initialBackoff: 10 * time.Second, want: 10 * time.Second}, + } + + for _, test := range tests { + max := 1024 * time.Second + backoff := newAgentBackoff(test.initialBackoff, max) + + // Test initial value + if backoff.current != test.want { + t.Fatalf("expected %d initial backoff, got: %v", test.want, backoff.current) + } + + // Test that backoff values are in expected range (75-100% of 2*previous) + for i := 0; i < 5; i++ { + old := backoff.current + backoff.next() + + expMax := 2 * old + expMin := 3 * expMax / 4 + + if backoff.current < expMin || backoff.current > expMax { + t.Fatalf("expected backoff in range %v to %v, got: %v", expMin, expMax, backoff) + } + } + + // Test that backoff is capped + for i := 0; i < 100; i++ { + backoff.next() + if backoff.current > max { + t.Fatalf("backoff exceeded max of 100s: %v", backoff) + } + } + + // Test reset + backoff.reset(test.initialBackoff) + if backoff.current != test.want { + t.Fatalf("expected %d backoff after reset, got: %v", test.want, backoff.current) + } + } +} From 02992c663fbff3c3a83d2b3613d4765922f1f982 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Wed, 27 Apr 2022 16:47:42 -0400 Subject: [PATCH 03/11] Fix bug --- command/agent.go | 1 + command/agent/auth/auth.go | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/command/agent.go b/command/agent.go index 93fffbf28cc4..e4d5ef263794 100644 --- a/command/agent.go +++ b/command/agent.go @@ -802,6 +802,7 @@ func (c *AgentCommand) Run(args []string) int { Logger: c.logger.Named("auth.handler"), Client: clonedClient, WrapTTL: config.AutoAuth.Method.WrapTTL, + InitialBackoff: config.AutoAuth.Method.InitialBackoff, MaxBackoff: config.AutoAuth.Method.MaxBackoff, EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials, EnableTemplateTokenCh: enableTokenCh, diff --git a/command/agent/auth/auth.go b/command/agent/auth/auth.go index c30391d83344..e9c63e38aed1 100644 --- a/command/agent/auth/auth.go +++ b/command/agent/auth/auth.go @@ -396,10 +396,6 @@ func (b *agentBackoff) next() { } func (b *agentBackoff) reset(initialBackoff time.Duration) { - if initialBackoff <= 0 { - initialBackoff = defaultInitialBackoff - } - b.current = initialBackoff } From 98a6b64c699bb3e3808378df573983d43a4103c6 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:08:12 -0400 Subject: [PATCH 04/11] Thread initial backoff to CT --- command/agent/template/template.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index cf21944ab0a8..90dc602a7100 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -311,6 +311,12 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) (*ctc Enabled: &enabled, } + if sc.AgentConfig.AutoAuth != nil && sc.AgentConfig.AutoAuth.Method != nil { + if sc.AgentConfig.AutoAuth.Method.InitialBackoff > 0 { + conf.Vault.Retry.Backoff = &sc.AgentConfig.AutoAuth.Method.InitialBackoff + } + } + conf.Finalize() // setup log level from TemplateServer config From 888a7f524cdbdc31f3a7de17c3074386c42d1626 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:10:13 -0400 Subject: [PATCH 05/11] Add comment --- command/agent/template/template.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index 90dc602a7100..63e707e2041f 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -311,6 +311,9 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) (*ctc Enabled: &enabled, } + // Sync Consul Template's retry with user set auto-auth initial backoff value. + // This is helpful if Auto Auth cannot get a new token and CT is trying to fetch + // secrets. if sc.AgentConfig.AutoAuth != nil && sc.AgentConfig.AutoAuth.Method != nil { if sc.AgentConfig.AutoAuth.Method.InitialBackoff > 0 { conf.Vault.Retry.Backoff = &sc.AgentConfig.AutoAuth.Method.InitialBackoff From baba936a9c75f8ef0454dca7878a8dd4594c64e1 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:21:33 -0400 Subject: [PATCH 06/11] Change to min_backoff --- command/agent.go | 2 +- command/agent/auth/auth.go | 18 ++++++------- command/agent/config/config.go | 26 +++++++++---------- command/agent/config/config_test.go | 10 +++---- .../config-method-initial-backoff.hcl | 2 +- command/agent/template/template.go | 4 +-- website/content/docs/agent/autoauth/index.mdx | 7 ++++- 7 files changed, 37 insertions(+), 32 deletions(-) diff --git a/command/agent.go b/command/agent.go index e4d5ef263794..2b848651573b 100644 --- a/command/agent.go +++ b/command/agent.go @@ -802,7 +802,7 @@ func (c *AgentCommand) Run(args []string) int { Logger: c.logger.Named("auth.handler"), Client: clonedClient, WrapTTL: config.AutoAuth.Method.WrapTTL, - InitialBackoff: config.AutoAuth.Method.InitialBackoff, + MinBackoff: config.AutoAuth.Method.MinBackoff, MaxBackoff: config.AutoAuth.Method.MaxBackoff, EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials, EnableTemplateTokenCh: enableTokenCh, diff --git a/command/agent/auth/auth.go b/command/agent/auth/auth.go index e9c63e38aed1..26cb6cced1b2 100644 --- a/command/agent/auth/auth.go +++ b/command/agent/auth/auth.go @@ -55,7 +55,7 @@ type AuthHandler struct { random *rand.Rand wrapTTL time.Duration maxBackoff time.Duration - initialBackoff time.Duration + minBackoff time.Duration enableReauthOnNewCredentials bool enableTemplateTokenCh bool } @@ -65,14 +65,14 @@ type AuthHandlerConfig struct { Client *api.Client WrapTTL time.Duration MaxBackoff time.Duration - InitialBackoff time.Duration + MinBackoff time.Duration Token string EnableReauthOnNewCredentials bool EnableTemplateTokenCh bool } func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { - initial := conf.InitialBackoff + initial := conf.MinBackoff if initial <= 0 { initial = defaultInitialBackoff } @@ -87,7 +87,7 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { client: conf.Client, random: rand.New(rand.NewSource(int64(time.Now().Nanosecond()))), wrapTTL: conf.WrapTTL, - initialBackoff: initial, + minBackoff: initial, maxBackoff: conf.MaxBackoff, enableReauthOnNewCredentials: conf.EnableReauthOnNewCredentials, enableTemplateTokenCh: conf.EnableTemplateTokenCh, @@ -112,11 +112,11 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { return errors.New("auth handler: nil auth method") } - if ah.initialBackoff <= 0 { - ah.initialBackoff = defaultInitialBackoff + if ah.minBackoff <= 0 { + ah.minBackoff = defaultInitialBackoff } - backoff := newAgentBackoff(ah.initialBackoff, ah.maxBackoff) + backoff := newAgentBackoff(ah.minBackoff, ah.maxBackoff) if backoff.current >= backoff.max { return errors.New("auth handler: initial backoff cannot be greater than max backoff") @@ -278,7 +278,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } am.CredSuccess() - backoff.reset(ah.initialBackoff) + backoff.reset(ah.minBackoff) select { case <-ctx.Done(): @@ -310,7 +310,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } am.CredSuccess() - backoff.reset(ah.initialBackoff) + backoff.reset(ah.minBackoff) } if watcher != nil { diff --git a/command/agent/config/config.go b/command/agent/config/config.go index 502b8d67f4dc..e68af26f644f 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -108,16 +108,16 @@ type AutoAuth struct { // Method represents the configuration for the authentication backend type Method struct { - Type string - MountPath string `hcl:"mount_path"` - WrapTTLRaw interface{} `hcl:"wrap_ttl"` - WrapTTL time.Duration `hcl:"-"` - InitialBackoffRaw interface{} `hcl:"initial_backoff"` - InitialBackoff time.Duration `hcl:"-"` - MaxBackoffRaw interface{} `hcl:"max_backoff"` - MaxBackoff time.Duration `hcl:"-"` - Namespace string `hcl:"namespace"` - Config map[string]interface{} + Type string + MountPath string `hcl:"mount_path"` + WrapTTLRaw interface{} `hcl:"wrap_ttl"` + WrapTTL time.Duration `hcl:"-"` + MinBackoffRaw interface{} `hcl:"min_backoff"` + MinBackoff time.Duration `hcl:"-"` + MaxBackoffRaw interface{} `hcl:"max_backoff"` + MaxBackoff time.Duration `hcl:"-"` + Namespace string `hcl:"namespace"` + Config map[string]interface{} } // Sink defines a location to write the authenticated token @@ -472,12 +472,12 @@ func parseAutoAuth(result *Config, list *ast.ObjectList) error { result.AutoAuth.Method.MaxBackoffRaw = nil } - if result.AutoAuth.Method.InitialBackoffRaw != nil { + if result.AutoAuth.Method.MinBackoffRaw != nil { var err error - if result.AutoAuth.Method.InitialBackoff, err = parseutil.ParseDurationSecond(result.AutoAuth.Method.InitialBackoffRaw); err != nil { + if result.AutoAuth.Method.MinBackoff, err = parseutil.ParseDurationSecond(result.AutoAuth.Method.MinBackoffRaw); err != nil { return err } - result.AutoAuth.Method.InitialBackoffRaw = nil + result.AutoAuth.Method.MinBackoffRaw = nil } return nil diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index 88569af72778..1a1aec2a14d1 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -302,11 +302,11 @@ func TestLoadConfigFile_Method_InitialBackoff(t *testing.T) { }, AutoAuth: &AutoAuth{ Method: &Method{ - Type: "aws", - MountPath: "auth/aws", - WrapTTL: 5 * time.Minute, - InitialBackoff: 5 * time.Second, - MaxBackoff: 2 * time.Minute, + Type: "aws", + MountPath: "auth/aws", + WrapTTL: 5 * time.Minute, + MinBackoff: 5 * time.Second, + MaxBackoff: 2 * time.Minute, Config: map[string]interface{}{ "role": "foobar", }, diff --git a/command/agent/config/test-fixtures/config-method-initial-backoff.hcl b/command/agent/config/test-fixtures/config-method-initial-backoff.hcl index d70959822dd0..6b9343aa4ba6 100644 --- a/command/agent/config/test-fixtures/config-method-initial-backoff.hcl +++ b/command/agent/config/test-fixtures/config-method-initial-backoff.hcl @@ -8,7 +8,7 @@ auto_auth { role = "foobar" } max_backoff = "2m" - initial_backoff = "5s" + min_backoff = "5s" } sink { diff --git a/command/agent/template/template.go b/command/agent/template/template.go index 63e707e2041f..8e13cfd4022d 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -315,8 +315,8 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) (*ctc // This is helpful if Auto Auth cannot get a new token and CT is trying to fetch // secrets. if sc.AgentConfig.AutoAuth != nil && sc.AgentConfig.AutoAuth.Method != nil { - if sc.AgentConfig.AutoAuth.Method.InitialBackoff > 0 { - conf.Vault.Retry.Backoff = &sc.AgentConfig.AutoAuth.Method.InitialBackoff + if sc.AgentConfig.AutoAuth.Method.MinBackoff > 0 { + conf.Vault.Retry.Backoff = &sc.AgentConfig.AutoAuth.Method.MinBackoff } } diff --git a/website/content/docs/agent/autoauth/index.mdx b/website/content/docs/agent/autoauth/index.mdx index 5a87359c809c..cd2f0e8d78b6 100644 --- a/website/content/docs/agent/autoauth/index.mdx +++ b/website/content/docs/agent/autoauth/index.mdx @@ -138,8 +138,13 @@ These are common configuration values that live within the `method` block: structure. Values can be an integer number of seconds or a stringish value like `5m`. +- `min_backoff` `(string or integer: "1s")` - The initial backoff time Agent + will delay before retrying after a failed auth attempt. The backoff will start + at the configured value and double (with some randomness) after successive + failures, capped by `max_backoff.` + - `max_backoff` `(string or integer: "5m")` - The maximum time Agent will delay - before retrying after a failed auth attempt. The backoff will start at 1 second + before retrying after a failed auth attempt. The backoff will start at `min_backoff` and double (with some randomness) after successive failures, capped by `max_backoff.` - `config` `(object: required)` - Configuration of the method itself. See the From 02429707d8e8d6fdd6f635b6f15b8154ad0170a7 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:30:53 -0400 Subject: [PATCH 07/11] changelog --- changelog/15204.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/15204.txt diff --git a/changelog/15204.txt b/changelog/15204.txt new file mode 100644 index 000000000000..f4c5b8cc4350 --- /dev/null +++ b/changelog/15204.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent/auto-auth: Add `min_backoff` to the method stanza for configuring initial backoff duration. +``` From ea9257894551dd728d0eeea605bd5c3f13e29aed Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Thu, 28 Apr 2022 10:47:53 -0400 Subject: [PATCH 08/11] remove initial references, review --- website/content/docs/agent/autoauth/index.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/content/docs/agent/autoauth/index.mdx b/website/content/docs/agent/autoauth/index.mdx index cd2f0e8d78b6..2883482b8e54 100644 --- a/website/content/docs/agent/autoauth/index.mdx +++ b/website/content/docs/agent/autoauth/index.mdx @@ -138,10 +138,11 @@ These are common configuration values that live within the `method` block: structure. Values can be an integer number of seconds or a stringish value like `5m`. -- `min_backoff` `(string or integer: "1s")` - The initial backoff time Agent +- `min_backoff` `(string or integer: "1s")` - The minimum backoff time Agent will delay before retrying after a failed auth attempt. The backoff will start at the configured value and double (with some randomness) after successive - failures, capped by `max_backoff.` + failures, capped by `max_backoff.` If Agent templating is being used, this + value is also used as the min backoff time for the templating server. - `max_backoff` `(string or integer: "5m")` - The maximum time Agent will delay before retrying after a failed auth attempt. The backoff will start at `min_backoff` From 89806c326ed77bcdf0d410080c95f856d25d8b31 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Thu, 28 Apr 2022 11:03:56 -0400 Subject: [PATCH 09/11] fix test --- command/agent/auth/auth.go | 35 +++++++++++++++------------------ command/agent/auth/auth_test.go | 26 ++++++++++++------------ 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/command/agent/auth/auth.go b/command/agent/auth/auth.go index 26cb6cced1b2..7406d7c8c846 100644 --- a/command/agent/auth/auth.go +++ b/command/agent/auth/auth.go @@ -15,8 +15,8 @@ import ( ) const ( - defaultInitialBackoff = 1 * time.Second - defaultMaxBackoff = 5 * time.Minute + defaultMinBackoff = 1 * time.Second + defaultMaxBackoff = 5 * time.Minute ) // AuthMethod is the interface that auto-auth methods implement for the agent @@ -72,11 +72,6 @@ type AuthHandlerConfig struct { } func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { - initial := conf.MinBackoff - if initial <= 0 { - initial = defaultInitialBackoff - } - ah := &AuthHandler{ // This is buffered so that if we try to output after the sink server // has been shut down, during agent shutdown, we won't block @@ -87,7 +82,7 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler { client: conf.Client, random: rand.New(rand.NewSource(int64(time.Now().Nanosecond()))), wrapTTL: conf.WrapTTL, - minBackoff: initial, + minBackoff: conf.MinBackoff, maxBackoff: conf.MaxBackoff, enableReauthOnNewCredentials: conf.EnableReauthOnNewCredentials, enableTemplateTokenCh: conf.EnableTemplateTokenCh, @@ -113,13 +108,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } if ah.minBackoff <= 0 { - ah.minBackoff = defaultInitialBackoff + ah.minBackoff = defaultMinBackoff } backoff := newAgentBackoff(ah.minBackoff, ah.maxBackoff) - if backoff.current >= backoff.max { - return errors.New("auth handler: initial backoff cannot be greater than max backoff") + if backoff.min >= backoff.max { + return errors.New("auth handler: min_backoff cannot be greater than max_backoff") } ah.logger.Info("starting auth handler") @@ -278,7 +273,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } am.CredSuccess() - backoff.reset(ah.minBackoff) + backoff.reset() select { case <-ctx.Done(): @@ -310,7 +305,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { } am.CredSuccess() - backoff.reset(ah.minBackoff) + backoff.reset() } if watcher != nil { @@ -362,22 +357,24 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error { // agentBackoff tracks exponential backoff state. type agentBackoff struct { + min time.Duration max time.Duration current time.Duration } -func newAgentBackoff(initial, max time.Duration) *agentBackoff { +func newAgentBackoff(min, max time.Duration) *agentBackoff { if max <= 0 { max = defaultMaxBackoff } - if initial <= 0 { - initial = defaultInitialBackoff + if min <= 0 { + min = defaultMinBackoff } return &agentBackoff{ + current: min, max: max, - current: initial, + min: min, } } @@ -395,8 +392,8 @@ func (b *agentBackoff) next() { b.current = maxBackoff - time.Duration(trim) } -func (b *agentBackoff) reset(initialBackoff time.Duration) { - b.current = initialBackoff +func (b *agentBackoff) reset() { + b.current = b.min } func (b agentBackoff) String() string { diff --git a/command/agent/auth/auth_test.go b/command/agent/auth/auth_test.go index 925a8b05f9bf..e0c2442d34fa 100644 --- a/command/agent/auth/auth_test.go +++ b/command/agent/auth/auth_test.go @@ -109,10 +109,10 @@ consumption: func TestAgentBackoff(t *testing.T) { max := 1024 * time.Second - backoff := newAgentBackoff(defaultInitialBackoff, max) + backoff := newAgentBackoff(defaultMinBackoff, max) // Test initial value - if backoff.current != defaultInitialBackoff { + if backoff.current != defaultMinBackoff { t.Fatalf("expected 1s initial backoff, got: %v", backoff.current) } @@ -138,28 +138,28 @@ func TestAgentBackoff(t *testing.T) { } // Test reset - backoff.reset(defaultInitialBackoff) - if backoff.current != defaultInitialBackoff { + backoff.reset() + if backoff.current != defaultMinBackoff { t.Fatalf("expected 1s backoff after reset, got: %v", backoff.current) } } -func TestAgentInitialBackoffCustom(t *testing.T) { +func TestAgentMinBackoffCustom(t *testing.T) { type test struct { - initialBackoff time.Duration - want time.Duration + minBackoff time.Duration + want time.Duration } tests := []test{ - {initialBackoff: 0 * time.Second, want: 1 * time.Second}, - {initialBackoff: 1 * time.Second, want: 1 * time.Second}, - {initialBackoff: 5 * time.Second, want: 5 * time.Second}, - {initialBackoff: 10 * time.Second, want: 10 * time.Second}, + {minBackoff: 0 * time.Second, want: 1 * time.Second}, + {minBackoff: 1 * time.Second, want: 1 * time.Second}, + {minBackoff: 5 * time.Second, want: 5 * time.Second}, + {minBackoff: 10 * time.Second, want: 10 * time.Second}, } for _, test := range tests { max := 1024 * time.Second - backoff := newAgentBackoff(test.initialBackoff, max) + backoff := newAgentBackoff(test.minBackoff, max) // Test initial value if backoff.current != test.want { @@ -188,7 +188,7 @@ func TestAgentInitialBackoffCustom(t *testing.T) { } // Test reset - backoff.reset(test.initialBackoff) + backoff.reset() if backoff.current != test.want { t.Fatalf("expected %d backoff after reset, got: %v", test.want, backoff.current) } From 0b64f5f2aa5acd72c505f279315ee7a67e6eb99c Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Thu, 28 Apr 2022 14:58:07 -0400 Subject: [PATCH 10/11] Thread max_backoff through --- command/agent/template/template.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/command/agent/template/template.go b/command/agent/template/template.go index 8e13cfd4022d..9ff22fbd9b25 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -318,6 +318,10 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) (*ctc if sc.AgentConfig.AutoAuth.Method.MinBackoff > 0 { conf.Vault.Retry.Backoff = &sc.AgentConfig.AutoAuth.Method.MinBackoff } + + if sc.AgentConfig.AutoAuth.Method.MaxBackoff > 0 { + conf.Vault.Retry.MaxBackoff = &sc.AgentConfig.AutoAuth.Method.MaxBackoff + } } conf.Finalize() From f702ae90729451598b4ce52a2b96263411722405 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Fri, 29 Apr 2022 11:53:29 -0400 Subject: [PATCH 11/11] Add doc note for max_backoff/templating --- website/content/docs/agent/autoauth/index.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/content/docs/agent/autoauth/index.mdx b/website/content/docs/agent/autoauth/index.mdx index 2883482b8e54..922c7427e51d 100644 --- a/website/content/docs/agent/autoauth/index.mdx +++ b/website/content/docs/agent/autoauth/index.mdx @@ -147,6 +147,8 @@ These are common configuration values that live within the `method` block: - `max_backoff` `(string or integer: "5m")` - The maximum time Agent will delay before retrying after a failed auth attempt. The backoff will start at `min_backoff` and double (with some randomness) after successive failures, capped by `max_backoff.` + If Agent templating is being used, this value is also used as the max backoff time + for the templating server. - `config` `(object: required)` - Configuration of the method itself. See the sidebar for information about each method.