Add NIST guidance on rotating keys used for AES-GCM encryption (#10612)

* Add NIST guidance on rotating keys used for AES-GCM encryption

* Capture more places barrier encryption is used

* spacing issue

* Probabilistically track an estimated encryption count by key term

* Un-reorder imports

* wip

* get rid of sampling

vault/barrier_aes_gcm.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -47,6 +48,8 @@ type barrierInit struct {
 // Validate AESGCMBarrier satisfies SecurityBarrier interface
 var _ SecurityBarrier = &AESGCMBarrier{}
 
+var barrierEncryptsMetric = []string{"barrier", "estimated_encryptions"}
+
 // AESGCMBarrier is a SecurityBarrier implementation that uses the AES
 // cipher core and the Galois Counter Mode block mode. It defaults to
 // the golang NONCE default value of 12 and a key size of 256
@@ -932,6 +935,8 @@ func (b *AESGCMBarrier) encrypt(path string, term uint32, gcm cipher.AEAD, plain
 		return nil, errors.New("unable to read enough random bytes to fill gcm nonce")
 	}
 
+	metrics.IncrCounterWithLabels(barrierEncryptsMetric, 1, termLabel(term))
+
 	// Seal the output
 	switch b.currentAESGCMVersionByte {
 	case AESGCMVersion1:
@@ -949,6 +954,15 @@ func (b *AESGCMBarrier) encrypt(path string, term uint32, gcm cipher.AEAD, plain
 	return out, nil
 }
 
+func termLabel(term uint32) []metrics.Label {
+	return []metrics.Label{
+		{
+			Name:  "term",
+			Value: strconv.FormatUint(uint64(term), 10),
+		},
+	}
+}
+
 // decrypt is used to decrypt a value using the keyring
 func (b *AESGCMBarrier) decrypt(path string, gcm cipher.AEAD, cipher []byte) ([]byte, error) {
 	// Capture the parts
@@ -972,7 +986,7 @@ func (b *AESGCMBarrier) decrypt(path string, gcm cipher.AEAD, cipher []byte) ([]
 }
 
 // Encrypt is used to encrypt in-memory for the BarrierEncryptor interface
-func (b *AESGCMBarrier) Encrypt(ctx context.Context, key string, plaintext []byte) ([]byte, error) {
+func (b *AESGCMBarrier) Encrypt(_ context.Context, key string, plaintext []byte) ([]byte, error) {
 	b.l.RLock()
 	if b.sealed {
 		b.l.RUnlock()
@@ -990,11 +1004,12 @@ func (b *AESGCMBarrier) Encrypt(ctx context.Context, key string, plaintext []byt
 	if err != nil {
 		return nil, err
 	}
+
 	return ciphertext, nil
 }
 
 // Decrypt is used to decrypt in-memory for the BarrierEncryptor interface
-func (b *AESGCMBarrier) Decrypt(ctx context.Context, key string, ciphertext []byte) ([]byte, error) {
+func (b *AESGCMBarrier) Decrypt(_ context.Context, key string, ciphertext []byte) ([]byte, error) {
 	b.l.RLock()
 	if b.sealed {
 		b.l.RUnlock()

vault/keyring.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/errwrap"
@@ -32,10 +33,12 @@ type EncodedKeyring struct {
 
 // Key represents a single term, along with the key used.
 type Key struct {
-	Term        uint32
-	Version     int
-	Value       []byte
-	InstallTime time.Time
+	Term                uint32
+	Version             int
+	Value               []byte
+	InstallTime         time.Time
+	Encryptions         uint64
+	ReportedEncryptions uint64 `json:",omitempty"`
 }
 
 // Serialize is used to create a byte encoded key
@@ -201,3 +204,10 @@ func (k *Keyring) Zeroize(keysToo bool) {
 		memzero(key.Value)
 	}
 }
+
+func (k *Keyring) AddEncryptionEstimate(term uint32, delta uint64) {
+	key := k.TermKey(term)
+	if key != nil {
+		atomic.AddUint64(&key.Encryptions, delta)
+	}
+}

website/pages/docs/internals/rotation.mdx

@@ -54,3 +54,21 @@ provides the `N+1` encryption key protected by the `N` key. This upgrade key is
 for a few minutes enabling standby instances to do a periodic check for upgrades.
 This allows standby instances to update their keys and stay in-sync with the active Vault
 without requiring operators to perform another unseal.
+
+## NIST Rotation Guidance
+
+Period rotation of the encryption keys is recommended, even in the absence of 
+compromise.  Due to the nature of the AES-256-GCM encryption used, keys should be 
+rotated before approximately 2<sup>32</sup> encryptions have been performed, following
+the guidelines of NIST publication 800-38D.  Operators can estimate the number
+of encryptions by summing the following:
+
+* The `vault.barrier.put` telemetry metric.
+* The `vault.token.creation` metric where the `token_type` label is `batch`.
+* The `merkle.flushDirty.num_pages` metric.
+* The WAL index.
+
+The simplest strategy may be to use those metrics to determine a frequency of 
+rotation and make that part of the operational process.  For example, if one 
+determines that the estimated rate is 40 million operations per day, then 
+rotating the key every three months is sufficient.  

website/pages/docs/secrets/transit/index.mdx

@@ -50,6 +50,17 @@ multiple of five) may provide a good alternative, allowing for several keys to
 be live at once and a deterministic way to decide which key to use at any given
 time.
 
+## NIST Rotation Guidance
+
+Period rotation of the encryption keys is recommended, even in the absence of 
+compromise.  For AES-GCM keys, rotation should occur before approximately 2<sup>32</sup> 
+encryptions have been performed by a key version, following the guidelines of NIST 
+publication 800-38D.  It is recommended that operators estimate the
+encryption rate of a key and use that to determine a frequency of rotation
+that prevents the guidance limits from being reached.  For example, if one determines 
+that the estimated rate is 40 million operations per day, then rotating a key every
+three months is sufficient.  
+
 ## Key Types
 
 As of now, the transit secrets engine supports the following key types (all key