From cc67cc9eda40a8df84e5d39cac0faca7ce8df43a Mon Sep 17 00:00:00 2001 From: Rory Geoghegan Date: Tue, 1 Mar 2022 16:48:11 -0500 Subject: [PATCH] configurable lease check wait for non-renewable secrets Now the vault config has a `lease_renewal_threshold` parameter which controls the fraction of how long of the original lease duration consul-template should wait to ask for a new secret on non-renewable secrets (like for PKIs). By default, consul-template will wait (90 +/- 5)% of the lease time. --- config/convert.go | 5 ++ config/vault.go | 22 +++++++- config/vault_test.go | 94 +++++++++++++++++++++++++++---- dependency/health_service_test.go | 1 + dependency/vault_common.go | 30 ++++++++-- dependency/vault_common_test.go | 1 + docs/configuration.md | 6 ++ manager/runner.go | 1 + 8 files changed, 142 insertions(+), 18 deletions(-) diff --git a/config/convert.go b/config/convert.go index 0fc45bddc..77f9b054a 100644 --- a/config/convert.go +++ b/config/convert.go @@ -71,6 +71,11 @@ func FileModePresent(o *os.FileMode) bool { return *o != 0 } +// Float64 returns a pointer to the given float64 +func Float64(f float64) *float64 { + return &f +} + // Int returns a pointer to the given int. func Int(i int) *int { return &i diff --git a/config/vault.go b/config/vault.go index 7c590d679..6f06f258a 100644 --- a/config/vault.go +++ b/config/vault.go @@ -30,6 +30,10 @@ const ( // DefaultVaultLeaseDuration is the default lease duration in seconds. DefaultVaultLeaseDuration = 5 * time.Minute + + // DefaultLeaseRenewalThreshold is the default fraction of a non-renewable + // lease to wait for before refreshing + DefaultLeaseRenewalThreshold = .90 ) // VaultConfig is the configuration for connecting to a vault server. @@ -74,6 +78,11 @@ type VaultConfig struct { // DefaultLeaseDuration configures the default lease duration when not explicitly // set by vault DefaultLeaseDuration *time.Duration `mapstructure:"default_lease_duration"` + + // LeaseRenewalThreshold configues how long Consul Template should wait for to + // refresh dynamic, non-renewable leases, measured as a fraction of the lease + // duration. + LeaseRenewalThreshold *float64 `mapstructure:"lease_renewal_threshold"` } // DefaultVaultConfig returns a configuration that is populated with the @@ -125,6 +134,7 @@ func (c *VaultConfig) Copy() *VaultConfig { o.UnwrapToken = c.UnwrapToken o.DefaultLeaseDuration = c.DefaultLeaseDuration + o.LeaseRenewalThreshold = c.LeaseRenewalThreshold return &o } @@ -191,6 +201,10 @@ func (c *VaultConfig) Merge(o *VaultConfig) *VaultConfig { r.DefaultLeaseDuration = o.DefaultLeaseDuration } + if o.LeaseRenewalThreshold != nil { + r.LeaseRenewalThreshold = o.LeaseRenewalThreshold + } + return r } @@ -292,6 +306,10 @@ func (c *VaultConfig) Finalize() { if c.DefaultLeaseDuration == nil { c.DefaultLeaseDuration = TimeDuration(DefaultVaultLeaseDuration) } + + if c.LeaseRenewalThreshold == nil { + c.LeaseRenewalThreshold = Float64(DefaultLeaseRenewalThreshold) + } } // GoString defines the printable version of this struct. @@ -310,8 +328,9 @@ func (c *VaultConfig) GoString() string { "Token:%t, "+ "VaultAgentTokenFile:%t, "+ "Transport:%#v, "+ - "UnwrapToken:%s"+ + "UnwrapToken:%s, "+ "DefaultLeaseDuration:%s, "+ + "LeaseRenewalThreshold:%f, "+ "}", StringGoString(c.Address), BoolGoString(c.Enabled), @@ -324,5 +343,6 @@ func (c *VaultConfig) GoString() string { c.Transport, BoolGoString(c.UnwrapToken), TimeDurationGoString(c.DefaultLeaseDuration), + *c.LeaseRenewalThreshold, ) } diff --git a/config/vault_test.go b/config/vault_test.go index 99b2de923..e338ce3e7 100644 --- a/config/vault_test.go +++ b/config/vault_test.go @@ -34,9 +34,10 @@ func TestVaultConfig_Copy(t *testing.T) { Transport: &TransportConfig{ DialKeepAlive: TimeDuration(20 * time.Second), }, - UnwrapToken: Bool(true), - VaultAgentTokenFile: String("/tmp/vault/agent/token"), - DefaultLeaseDuration: TimeDuration(5 * time.Minute), + UnwrapToken: Bool(true), + VaultAgentTokenFile: String("/tmp/vault/agent/token"), + DefaultLeaseDuration: TimeDuration(5 * time.Minute), + LeaseRenewalThreshold: Float64(0.70), }, }, } @@ -323,6 +324,30 @@ func TestVaultConfig_Merge(t *testing.T) { &VaultConfig{DefaultLeaseDuration: TimeDuration(5 * time.Minute)}, &VaultConfig{DefaultLeaseDuration: TimeDuration(5 * time.Minute)}, }, + { + "lease_renewal_threshold_overrides", + &VaultConfig{LeaseRenewalThreshold: Float64(0.8)}, + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + }, + { + "lease_renewal_threshold_empty_one", + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + &VaultConfig{}, + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + }, + { + "lease_renewal_threshold_empty_two", + &VaultConfig{}, + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + }, + { + "lease_renewal_threshold_same", + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + &VaultConfig{LeaseRenewalThreshold: Float64(0.7)}, + }, } for i, tc := range cases { @@ -375,8 +400,9 @@ func TestVaultConfig_Finalize(t *testing.T) { MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, - UnwrapToken: Bool(DefaultVaultUnwrapToken), - DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration), + UnwrapToken: Bool(DefaultVaultUnwrapToken), + DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration), + LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold), }, }, { @@ -414,8 +440,9 @@ func TestVaultConfig_Finalize(t *testing.T) { MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, - UnwrapToken: Bool(DefaultVaultUnwrapToken), - DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration), + UnwrapToken: Bool(DefaultVaultUnwrapToken), + DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration), + LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold), }, }, { @@ -453,14 +480,15 @@ func TestVaultConfig_Finalize(t *testing.T) { MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, - UnwrapToken: Bool(DefaultVaultUnwrapToken), - DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration), + UnwrapToken: Bool(DefaultVaultUnwrapToken), + DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration), + LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold), }, }, { "with_default_lease_duration", &VaultConfig{ - Address: String("address"), + Address: String("address"), DefaultLeaseDuration: TimeDuration(1 * time.Minute), }, &VaultConfig{ @@ -493,8 +521,50 @@ func TestVaultConfig_Finalize(t *testing.T) { MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), }, - UnwrapToken: Bool(DefaultVaultUnwrapToken), - DefaultLeaseDuration: TimeDuration(1 * time.Minute), + UnwrapToken: Bool(DefaultVaultUnwrapToken), + DefaultLeaseDuration: TimeDuration(1 * time.Minute), + LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold), + }, + }, + { + "with_lease_renewal_threshold", + &VaultConfig{ + Address: String("address"), + LeaseRenewalThreshold: Float64(0.70), + }, + &VaultConfig{ + Address: String("address"), + Enabled: Bool(true), + Namespace: String(""), + RenewToken: Bool(false), + Retry: &RetryConfig{ + Backoff: TimeDuration(DefaultRetryBackoff), + MaxBackoff: TimeDuration(DefaultRetryMaxBackoff), + Enabled: Bool(true), + Attempts: Int(DefaultRetryAttempts), + }, + SSL: &SSLConfig{ + CaCert: String(""), + CaPath: String(""), + Cert: String(""), + Enabled: Bool(true), + Key: String(""), + ServerName: String(""), + Verify: Bool(true), + }, + Token: String(""), + Transport: &TransportConfig{ + DialKeepAlive: TimeDuration(DefaultDialKeepAlive), + DialTimeout: TimeDuration(DefaultDialTimeout), + DisableKeepAlives: Bool(false), + IdleConnTimeout: TimeDuration(DefaultIdleConnTimeout), + MaxIdleConns: Int(DefaultMaxIdleConns), + MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost), + TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout), + }, + UnwrapToken: Bool(DefaultVaultUnwrapToken), + DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration), + LeaseRenewalThreshold: Float64(0.70), }, }, } diff --git a/dependency/health_service_test.go b/dependency/health_service_test.go index c7b438dce..7bd38fa28 100644 --- a/dependency/health_service_test.go +++ b/dependency/health_service_test.go @@ -210,6 +210,7 @@ func TestHealthConnectServiceQuery_Fetch(t *testing.T) { inst.Node, inst.NodeID = "", "" inst.Checks = nil inst.NodeTaggedAddresses = nil + inst.ServiceTaggedAddresses = nil assert.Equal(t, tc.exp, act) }) diff --git a/dependency/vault_common.go b/dependency/vault_common.go index d1fd123e2..a99033bd0 100644 --- a/dependency/vault_common.go +++ b/dependency/vault_common.go @@ -13,8 +13,10 @@ import ( var ( // VaultDefaultLeaseDuration is the default lease duration in seconds. - VaultDefaultLeaseDuration time.Duration - onceVaultDefaultLeaseDuration sync.Once + VaultDefaultLeaseDuration time.Duration + onceVaultDefaultLeaseDuration sync.Once + VaultLeaseRenewalThreshold float64 + onceVaultLeaseRenewalThreshold sync.Once ) // Secret is the structure returned for every secret within Vault. @@ -166,9 +168,19 @@ func leaseCheckWait(s *Secret) time.Duration { // If the secret doesn't have a rotation period, this is a non-renewable leased // secret. // For non-renewable leases set the renew duration to use much of the secret - // lease as possible. Use a stagger over 85%-95% of the lease duration so that - // many clients do not hit Vault simultaneously. - sleep = sleep * (.85 + rand.Float64()*0.1) + // lease as possible. Use a stagger over the configured threshold + // fraction of the lease duration so that many clients do not hit + // Vault simultaneously. + finalFraction := VaultLeaseRenewalThreshold + (rand.Float64()-0.5)*0.1 + if finalFraction >= 1.0 || finalFraction <= 0.0 { + // If the fraction randomly winds up outside of (0.0-1.0), clamp + // back down to the VaultLeaseRenewalThreshold provided by the user, + // since a) the user picked that value, so they should be + // comfortable with it, and b) it should not skew the staggering too + // much + finalFraction = VaultLeaseRenewalThreshold + } + sleep = sleep * finalFraction } return time.Duration(sleep) @@ -351,3 +363,11 @@ func SetVaultDefaultLeaseDuration(t time.Duration) { } onceVaultDefaultLeaseDuration.Do(set) } + +// Make sure to only set VaultLeaseRenewalThreshold once +func SetVaultLeaseRenewalThreshold(f float64) { + set := func() { + VaultLeaseRenewalThreshold = f + } + onceVaultLeaseRenewalThreshold.Do(set) +} diff --git a/dependency/vault_common_test.go b/dependency/vault_common_test.go index 35ff63f4b..1cec33206 100644 --- a/dependency/vault_common_test.go +++ b/dependency/vault_common_test.go @@ -9,6 +9,7 @@ import ( func init() { VaultDefaultLeaseDuration = 0 + VaultLeaseRenewalThreshold = .90 } func TestVaultRenewDuration(t *testing.T) { diff --git a/docs/configuration.md b/docs/configuration.md index 406a6785b..6c1c15943 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -390,6 +390,12 @@ vault { # 5 minutes. default_lease_duration = "60s" + # The fraction of the lease duration of a non-renewable secret Consul + # Template will wait for. This is used to calculate the sleep duration for + # rechecking a Vault secret value. This field is optional and will default to + # 90% of the lease time. + default_lease_duration = 0.90 + # This option tells Consul Template to automatically renew the Vault token # given. If you are unfamiliar with Vault's architecture, Vault requires # tokens be renewed at some regular interval or they will be revoked. Consul diff --git a/manager/runner.go b/manager/runner.go index c3f40ce4f..f560628ea 100644 --- a/manager/runner.go +++ b/manager/runner.go @@ -882,6 +882,7 @@ func (r *Runner) init() error { log.Printf("[DEBUG] (runner) final config: %s", result) dep.SetVaultDefaultLeaseDuration(config.TimeDurationVal(r.config.Vault.DefaultLeaseDuration)) + dep.SetVaultLeaseRenewalThreshold(*r.config.Vault.LeaseRenewalThreshold) // Create the clientset clients, err := newClientSet(r.config)