configurable lease check wait for non-renewable secrets

Now the vault config has a `lease_renewal_threshold` parameter which controls the
fraction of how long of the original lease duration consul-template should wait to
ask for a new secret on non-renewable secrets (like for PKIs). By default, consul-template
will wait (90 +/- 5)% of the lease time.

config/convert.go

@@ -71,6 +71,11 @@ func FileModePresent(o *os.FileMode) bool {
 	return *o != 0
 }
 
+// Float64 returns a pointer to the given float64
+func Float64(f float64) *float64 {
+	return &f
+}
+
 // Int returns a pointer to the given int.
 func Int(i int) *int {
 	return &i

config/vault.go

@@ -30,6 +30,10 @@ const (
 
 	// DefaultVaultLeaseDuration is the default lease duration in seconds.
 	DefaultVaultLeaseDuration = 5 * time.Minute
+
+	// DefaultLeaseRenewalThreshold is the default fraction of a non-renewable
+	// lease to wait for before refreshing
+	DefaultLeaseRenewalThreshold = .90
 )
 
 // VaultConfig is the configuration for connecting to a vault server.
@@ -74,6 +78,11 @@ type VaultConfig struct {
 	// DefaultLeaseDuration configures the default lease duration when not explicitly
 	// set by vault
 	DefaultLeaseDuration *time.Duration `mapstructure:"default_lease_duration"`
+
+	// LeaseRenewalThreshold configues how long Consul Template should wait for to
+	// refresh dynamic, non-renewable leases, measured as a fraction of the lease
+	// duration.
+	LeaseRenewalThreshold *float64 `mapstructure:"lease_renewal_threshold"`
 }
 
 // DefaultVaultConfig returns a configuration that is populated with the
@@ -125,6 +134,7 @@ func (c *VaultConfig) Copy() *VaultConfig {
 	o.UnwrapToken = c.UnwrapToken
 
 	o.DefaultLeaseDuration = c.DefaultLeaseDuration
+	o.LeaseRenewalThreshold = c.LeaseRenewalThreshold
 
 	return &o
 }
@@ -191,6 +201,10 @@ func (c *VaultConfig) Merge(o *VaultConfig) *VaultConfig {
 		r.DefaultLeaseDuration = o.DefaultLeaseDuration
 	}
 
+	if o.LeaseRenewalThreshold != nil {
+		r.LeaseRenewalThreshold = o.LeaseRenewalThreshold
+	}
+
 	return r
 }
 
@@ -292,6 +306,10 @@ func (c *VaultConfig) Finalize() {
 	if c.DefaultLeaseDuration == nil {
 		c.DefaultLeaseDuration = TimeDuration(DefaultVaultLeaseDuration)
 	}
+
+	if c.LeaseRenewalThreshold == nil {
+		c.LeaseRenewalThreshold = Float64(DefaultLeaseRenewalThreshold)
+	}
 }
 
 // GoString defines the printable version of this struct.
@@ -310,8 +328,9 @@ func (c *VaultConfig) GoString() string {
 		"Token:%t, "+
 		"VaultAgentTokenFile:%t, "+
 		"Transport:%#v, "+
-		"UnwrapToken:%s"+
+		"UnwrapToken:%s, "+
 		"DefaultLeaseDuration:%s, "+
+		"LeaseRenewalThreshold:%f, "+
 		"}",
 		StringGoString(c.Address),
 		BoolGoString(c.Enabled),
@@ -324,5 +343,6 @@ func (c *VaultConfig) GoString() string {
 		c.Transport,
 		BoolGoString(c.UnwrapToken),
 		TimeDurationGoString(c.DefaultLeaseDuration),
+		*c.LeaseRenewalThreshold,
 	)
 }

config/vault_test.go

@@ -34,9 +34,10 @@ func TestVaultConfig_Copy(t *testing.T) {
 				Transport: &TransportConfig{
 					DialKeepAlive: TimeDuration(20 * time.Second),
 				},
-				UnwrapToken:         Bool(true),
-				VaultAgentTokenFile: String("/tmp/vault/agent/token"),
-				DefaultLeaseDuration: TimeDuration(5 * time.Minute),
+				UnwrapToken:           Bool(true),
+				VaultAgentTokenFile:   String("/tmp/vault/agent/token"),
+				DefaultLeaseDuration:  TimeDuration(5 * time.Minute),
+				LeaseRenewalThreshold: Float64(0.70),
 			},
 		},
 	}
@@ -323,6 +324,30 @@ func TestVaultConfig_Merge(t *testing.T) {
 			&VaultConfig{DefaultLeaseDuration: TimeDuration(5 * time.Minute)},
 			&VaultConfig{DefaultLeaseDuration: TimeDuration(5 * time.Minute)},
 		},
+		{
+			"lease_renewal_threshold_overrides",
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.8)},
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+		},
+		{
+			"lease_renewal_threshold_empty_one",
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+			&VaultConfig{},
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+		},
+		{
+			"lease_renewal_threshold_empty_two",
+			&VaultConfig{},
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+		},
+		{
+			"lease_renewal_threshold_same",
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+			&VaultConfig{LeaseRenewalThreshold: Float64(0.7)},
+		},
 	}
 
 	for i, tc := range cases {
@@ -375,8 +400,9 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
-				UnwrapToken: Bool(DefaultVaultUnwrapToken),
-				DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration),
+				UnwrapToken:           Bool(DefaultVaultUnwrapToken),
+				DefaultLeaseDuration:  TimeDuration(DefaultVaultLeaseDuration),
+				LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold),
 			},
 		},
 		{
@@ -414,8 +440,9 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
-				UnwrapToken: Bool(DefaultVaultUnwrapToken),
-				DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration),
+				UnwrapToken:           Bool(DefaultVaultUnwrapToken),
+				DefaultLeaseDuration:  TimeDuration(DefaultVaultLeaseDuration),
+				LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold),
 			},
 		},
 		{
@@ -453,14 +480,15 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
-				UnwrapToken: Bool(DefaultVaultUnwrapToken),
-				DefaultLeaseDuration: TimeDuration(DefaultVaultLeaseDuration),
+				UnwrapToken:           Bool(DefaultVaultUnwrapToken),
+				DefaultLeaseDuration:  TimeDuration(DefaultVaultLeaseDuration),
+				LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold),
 			},
 		},
 		{
 			"with_default_lease_duration",
 			&VaultConfig{
-				Address: String("address"),
+				Address:              String("address"),
 				DefaultLeaseDuration: TimeDuration(1 * time.Minute),
 			},
 			&VaultConfig{
@@ -493,8 +521,50 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
-				UnwrapToken: Bool(DefaultVaultUnwrapToken),
-				DefaultLeaseDuration: TimeDuration(1 * time.Minute),
+				UnwrapToken:           Bool(DefaultVaultUnwrapToken),
+				DefaultLeaseDuration:  TimeDuration(1 * time.Minute),
+				LeaseRenewalThreshold: Float64(DefaultLeaseRenewalThreshold),
+			},
+		},
+		{
+			"with_lease_renewal_threshold",
+			&VaultConfig{
+				Address:               String("address"),
+				LeaseRenewalThreshold: Float64(0.70),
+			},
+			&VaultConfig{
+				Address:    String("address"),
+				Enabled:    Bool(true),
+				Namespace:  String(""),
+				RenewToken: Bool(false),
+				Retry: &RetryConfig{
+					Backoff:    TimeDuration(DefaultRetryBackoff),
+					MaxBackoff: TimeDuration(DefaultRetryMaxBackoff),
+					Enabled:    Bool(true),
+					Attempts:   Int(DefaultRetryAttempts),
+				},
+				SSL: &SSLConfig{
+					CaCert:     String(""),
+					CaPath:     String(""),
+					Cert:       String(""),
+					Enabled:    Bool(true),
+					Key:        String(""),
+					ServerName: String(""),
+					Verify:     Bool(true),
+				},
+				Token: String(""),
+				Transport: &TransportConfig{
+					DialKeepAlive:       TimeDuration(DefaultDialKeepAlive),
+					DialTimeout:         TimeDuration(DefaultDialTimeout),
+					DisableKeepAlives:   Bool(false),
+					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
+					MaxIdleConns:        Int(DefaultMaxIdleConns),
+					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
+				},
+				UnwrapToken:           Bool(DefaultVaultUnwrapToken),
+				DefaultLeaseDuration:  TimeDuration(DefaultVaultLeaseDuration),
+				LeaseRenewalThreshold: Float64(0.70),
 			},
 		},
 	}

dependency/health_service_test.go

@@ -210,6 +210,7 @@ func TestHealthConnectServiceQuery_Fetch(t *testing.T) {
 			inst.Node, inst.NodeID = "", ""
 			inst.Checks = nil
 			inst.NodeTaggedAddresses = nil
+			inst.ServiceTaggedAddresses = nil
 
 			assert.Equal(t, tc.exp, act)
 		})

dependency/vault_common.go

@@ -13,8 +13,10 @@ import (
 
 var (
 	// VaultDefaultLeaseDuration is the default lease duration in seconds.
-	VaultDefaultLeaseDuration     time.Duration
-	onceVaultDefaultLeaseDuration sync.Once
+	VaultDefaultLeaseDuration      time.Duration
+	onceVaultDefaultLeaseDuration  sync.Once
+	VaultLeaseRenewalThreshold     float64
+	onceVaultLeaseRenewalThreshold sync.Once
 )
 
 // Secret is the structure returned for every secret within Vault.
@@ -166,9 +168,19 @@ func leaseCheckWait(s *Secret) time.Duration {
 		// If the secret doesn't have a rotation period, this is a non-renewable leased
 		// secret.
 		// For non-renewable leases set the renew duration to use much of the secret
-		// lease as possible. Use a stagger over 85%-95% of the lease duration so that
-		// many clients do not hit Vault simultaneously.
-		sleep = sleep * (.85 + rand.Float64()*0.1)
+		// lease as possible. Use a stagger over the configured threshold
+		// fraction of the lease duration so that many clients do not hit
+		// Vault simultaneously.
+		finalFraction := VaultLeaseRenewalThreshold + (rand.Float64()-0.5)*0.1
+		if finalFraction >= 1.0 || finalFraction <= 0.0 {
+			// If the fraction randomly winds up outside of (0.0-1.0), clamp
+			// back down to the VaultLeaseRenewalThreshold provided by the user,
+			// since a) the user picked that value, so they should be
+			// comfortable with it, and b) it should not skew the staggering too
+			// much
+			finalFraction = VaultLeaseRenewalThreshold
+		}
+		sleep = sleep * finalFraction
 	}
 
 	return time.Duration(sleep)
@@ -351,3 +363,11 @@ func SetVaultDefaultLeaseDuration(t time.Duration) {
 	}
 	onceVaultDefaultLeaseDuration.Do(set)
 }
+
+// Make sure to only set VaultLeaseRenewalThreshold once
+func SetVaultLeaseRenewalThreshold(f float64) {
+	set := func() {
+		VaultLeaseRenewalThreshold = f
+	}
+	onceVaultLeaseRenewalThreshold.Do(set)
+}

dependency/vault_common_test.go

@@ -9,6 +9,7 @@ import (
 
 func init() {
 	VaultDefaultLeaseDuration = 0
+	VaultLeaseRenewalThreshold = .90
 }
 
 func TestVaultRenewDuration(t *testing.T) {

docs/configuration.md

@@ -390,6 +390,12 @@ vault {
   # 5 minutes.
   default_lease_duration = "60s"
 
+  # The fraction of the lease duration of a non-renewable secret Consul
+  # Template will wait for. This is used to calculate the sleep duration for
+  # rechecking a Vault secret value. This field is optional and will default to
+  # 90% of the lease time.
+  default_lease_duration = 0.90
+
   # This option tells Consul Template to automatically renew the Vault token
   # given. If you are unfamiliar with Vault's architecture, Vault requires
   # tokens be renewed at some regular interval or they will be revoked. Consul

manager/runner.go

@@ -882,6 +882,7 @@ func (r *Runner) init() error {
 	log.Printf("[DEBUG] (runner) final config: %s", result)
 
 	dep.SetVaultDefaultLeaseDuration(config.TimeDurationVal(r.config.Vault.DefaultLeaseDuration))
+	dep.SetVaultLeaseRenewalThreshold(*r.config.Vault.LeaseRenewalThreshold)
 
 	// Create the clientset
 	clients, err := newClientSet(r.config)