Adding a callback based method to the secrets component

The secrets component can now notify the caller when resolving a secret.

This allows the config package to only overwrite the setting using
secrets instead of the entire configuration.

comp/core/secrets/component.go

@@ -18,6 +18,9 @@ type Component interface {
 	Configure(command string, arguments []string, timeout, maxSize int, groupExecPerm, removeLinebreak bool)
 	// Get debug information and write it to the parameter
 	GetDebugInfo(w io.Writer)
-	// Decrypt the given handle and return the corresponding secret value
-	Decrypt(data []byte, origin string) ([]byte, error)
+	// Resolve resolves the secrets in the given yaml data by replacing secrets handles by their corresponding secret value
+	Resolve(data []byte, origin string) ([]byte, error)
+	// ResolveWithCallback resolves the secrets in the given yaml data calling the callback with the YAML path of
+	// the secret handle and its value
+	ResolveWithCallback(data []byte, origin string, callback ResolveCallback) error
 }

comp/core/secrets/secretsimpl/fetch_secret.go

@@ -133,19 +133,19 @@ func (r *secretResolver) fetchSecret(secretsHandle []string) (map[string]string,
 	for _, sec := range secretsHandle {
 		v, ok := secrets[sec]
 		if !ok {
-			return nil, fmt.Errorf("secret handle '%s' was not decrypted by the secret_backend_command", sec)
+			return nil, fmt.Errorf("secret handle '%s' was not resolved by the secret_backend_command", sec)
 		}
 
 		if v.ErrorMsg != "" {
-			return nil, fmt.Errorf("an error occurred while decrypting '%s': %s", sec, v.ErrorMsg)
+			return nil, fmt.Errorf("an error occurred while resolving '%s': %s", sec, v.ErrorMsg)
 		}
 
 		if r.removeTrailingLinebreak {
 			v.Value = strings.TrimRight(v.Value, "\r\n")
 		}
 
 		if v.Value == "" {
-			return nil, fmt.Errorf("decrypted secret for '%s' is empty", sec)
+			return nil, fmt.Errorf("resolved secret for '%s' is empty", sec)
 		}
 
 		// add it to the cache

comp/core/secrets/secretsimpl/fetch_secret_test.go

@@ -201,7 +201,7 @@ func TestFetchSecretMissingSecret(t *testing.T) {
 	resolver.commandHookFunc = func(string) ([]byte, error) { return []byte("{}"), nil }
 	_, err := resolver.fetchSecret(secrets)
 	assert.NotNil(t, err)
-	assert.Equal(t, "secret handle 'handle1' was not decrypted by the secret_backend_command", err.Error())
+	assert.Equal(t, "secret handle 'handle1' was not resolved by the secret_backend_command", err.Error())
 }
 
 func TestFetchSecretErrorForHandle(t *testing.T) {
@@ -211,7 +211,7 @@ func TestFetchSecretErrorForHandle(t *testing.T) {
 	}
 	_, err := resolver.fetchSecret([]string{"handle1"})
 	assert.NotNil(t, err)
-	assert.Equal(t, "an error occurred while decrypting 'handle1': some error", err.Error())
+	assert.Equal(t, "an error occurred while resolving 'handle1': some error", err.Error())
 }
 
 func TestFetchSecretEmptyValue(t *testing.T) {
@@ -221,14 +221,14 @@ func TestFetchSecretEmptyValue(t *testing.T) {
 	}
 	_, err := resolver.fetchSecret([]string{"handle1"})
 	assert.NotNil(t, err)
-	assert.Equal(t, "decrypted secret for 'handle1' is empty", err.Error())
+	assert.Equal(t, "resolved secret for 'handle1' is empty", err.Error())
 
 	resolver.commandHookFunc = func(string) ([]byte, error) {
 		return []byte("{\"handle1\":{\"value\": \"\"}}"), nil
 	}
 	_, err = resolver.fetchSecret([]string{"handle1"})
 	assert.NotNil(t, err)
-	assert.Equal(t, "decrypted secret for 'handle1' is empty", err.Error())
+	assert.Equal(t, "resolved secret for 'handle1' is empty", err.Error())
 }
 
 func TestFetchSecret(t *testing.T) {

comp/core/secrets/secretsimpl/info.tmpl

@@ -10,8 +10,8 @@ Permissions Detail:
 {{- end }}
 
 === Secrets stats ===
-Number of secrets decrypted: {{ len .Handles }}
-Secrets handle decrypted:
+Number of secrets resolved: {{ len .Handles }}
+Secrets handle resolved:
 {{ range $handle, $places := .Handles }}
 - '{{ $handle }}':
 	{{- range $place := $places }}

comp/core/secrets/secretsimpl/info_nix_test.go

@@ -77,9 +77,9 @@ func TestDebugInfo(t *testing.T) {
 		return res, nil
 	}
 
-	_, err := resolver.Decrypt(testConf, "test")
+	_, err := resolver.Resolve(testConf, "test")
 	require.NoError(t, err)
-	_, err = resolver.Decrypt(testConfInfo, "test2")
+	_, err = resolver.Resolve(testConfInfo, "test2")
 	require.NoError(t, err)
 
 	var buffer bytes.Buffer
@@ -95,8 +95,8 @@ Owner: ` + currentUser + `
 Group: ` + currentGroup + `
 
 === Secrets stats ===
-Number of secrets decrypted: 3
-Secrets handle decrypted:
+Number of secrets resolved: 3
+Secrets handle resolved:
 
 - 'pass1':
 	used in 'test' configuration in entry 'instances/password'
@@ -121,9 +121,9 @@ func TestDebugInfoError(t *testing.T) {
 		return res, nil
 	}
 
-	_, err := resolver.Decrypt(testConf, "test")
+	_, err := resolver.Resolve(testConf, "test")
 	require.NoError(t, err)
-	_, err = resolver.Decrypt(testConfInfo, "test2")
+	_, err = resolver.Resolve(testConfInfo, "test2")
 	require.NoError(t, err)
 
 	var buffer bytes.Buffer
@@ -137,8 +137,8 @@ Permissions Detail:
 Could not stat some_command: no such file or directory
 
 === Secrets stats ===
-Number of secrets decrypted: 3
-Secrets handle decrypted:
+Number of secrets resolved: 3
+Secrets handle resolved:
 
 - 'pass1':
 	used in 'test' configuration in entry 'instances/password'

comp/core/secrets/secretsimpl/secrets.go

@@ -186,75 +186,6 @@ func (r *secretResolver) Configure(command string, arguments []string, timeout,
 	}
 }
 
-type walkerCallback func([]string, string) (string, error)
-
-func walkSlice(data []interface{}, yamlPath []string, callback walkerCallback) error {
-	for idx, k := range data {
-		switch v := k.(type) {
-		case string:
-			newValue, err := callback(yamlPath, v)
-			if err != nil {
-				return err
-			}
-			data[idx] = newValue
-		case map[interface{}]interface{}:
-			if err := walkHash(v, yamlPath, callback); err != nil {
-				return err
-			}
-		case []interface{}:
-			if err := walkSlice(v, yamlPath, callback); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func walkHash(data map[interface{}]interface{}, yamlPath []string, callback walkerCallback) error {
-	for k := range data {
-		path := yamlPath
-		if newkey, ok := k.(string); ok {
-			path = append(path, newkey)
-		}
-
-		switch v := data[k].(type) {
-		case string:
-			newValue, err := callback(path, v)
-			if err != nil {
-				return err
-			}
-			data[k] = newValue
-		case map[interface{}]interface{}:
-			if err := walkHash(v, path, callback); err != nil {
-				return err
-			}
-		case []interface{}:
-			if err := walkSlice(v, path, callback); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-// walk will go through loaded yaml and call callback on every strings allowing
-// the callback to overwrite the string value
-func walk(data *interface{}, yamlPath []string, callback walkerCallback) error {
-	switch v := (*data).(type) {
-	case string:
-		newValue, err := callback(yamlPath, v)
-		if err != nil {
-			return err
-		}
-		*data = newValue
-	case map[interface{}]interface{}:
-		return walkHash(v, yamlPath, callback)
-	case []interface{}:
-		return walkSlice(v, yamlPath, callback)
-	}
-	return nil
-}
-
 func isEnc(str string) (bool, string) {
 	// trimming space and tabs
 	str = strings.Trim(str, " 	")
@@ -264,9 +195,20 @@ func isEnc(str string) (bool, string) {
 	return false, ""
 }
 
-// Decrypt replaces all encrypted secrets in data by executing
-// "secret_backend_command" once if all secrets aren't present in the cache.
-func (r *secretResolver) Decrypt(data []byte, origin string) ([]byte, error) {
+// Resolve replaces all encrypted secrets in data by executing "secret_backend_command" once if all secrets aren't
+// present in the cache.
+func (r *secretResolver) Resolve(data []byte, origin string) ([]byte, error) {
+	return r.resolve(data, origin, nil)
+}
+
+// ResolveWithCallback resolves the secrets in the given yaml data calling the callback with the YAML path of
+// the secret handle and its value
+func (r *secretResolver) ResolveWithCallback(data []byte, origin string, cb secrets.ResolveCallback) error {
+	_, err := r.resolve(data, origin, cb)
+	return err
+}
+
+func (r *secretResolver) resolve(data []byte, origin string, notifyCb secrets.ResolveCallback) ([]byte, error) {
 	if !r.enabled {
 		log.Infof("Agent secrets is disabled by caller")
 		return nil, nil
@@ -284,11 +226,10 @@ func (r *secretResolver) Decrypt(data []byte, origin string) ([]byte, error) {
 	// First we collect all new handles in the config
 	newHandles := []string{}
 	haveSecret := false
-	err = walk(
-		&config,
-		nil,
-		func(yamlPath []string, str string) (string, error) {
-			if ok, handle := isEnc(str); ok {
+
+	w := &walker{
+		resolver: func(yamlPath []string, value string) (string, error) {
+			if ok, handle := isEnc(value); ok {
 				haveSecret = true
 				// Check if we already know this secret
 				if secret, ok := r.cache[handle]; ok {
@@ -298,10 +239,14 @@ func (r *secretResolver) Decrypt(data []byte, origin string) ([]byte, error) {
 					return secret, nil
 				}
 				newHandles = append(newHandles, handle)
+				return value, nil
 			}
-			return str, nil
-		})
-	if err != nil {
+			return value, nil
+		},
+		notifier: notifyCb,
+	}
+
+	if err := w.walk(&config); err != nil {
 		return nil, err
 	}
 
@@ -324,25 +269,24 @@ func (r *secretResolver) Decrypt(data []byte, origin string) ([]byte, error) {
 			return nil, err
 		}
 
-		// Replace all new encrypted secrets in the config
-		err = walk(
-			&config,
-			nil,
-			func(yamlPath []string, str string) (string, error) {
-				if ok, handle := isEnc(str); ok {
-					if secret, ok := secrets[handle]; ok {
-						log.Debugf("Secret '%s' was retrieved from executable", handle)
-						// keep track of place where a handle was found
-						r.registerSecretOrigin(handle, origin, yamlPath)
-						return secret, nil
-					}
-					// This should never happen since fetchSecret will return an error
-					// if not every handles have been fetched.
-					return str, fmt.Errorf("unknown secret '%s'", handle)
+		w.resolver = func(yamlPath []string, value string) (string, error) {
+			if ok, handle := isEnc(value); ok {
+				if secret, ok := secrets[handle]; ok {
+					log.Debugf("Secret '%s' was successfully resolved", handle)
+					// keep track of place where a handle was found
+					r.registerSecretOrigin(handle, origin, yamlPath)
+					return secret, nil
 				}
-				return str, nil
-			})
-		if err != nil {
+
+				// This should never happen since fetchSecret will return an error if not every handle have
+				// been fetched.
+				return "", fmt.Errorf("unknown secret '%s'", handle)
+			}
+			return value, nil
+		}
+
+		// Replace all newly resolved secrets in the config
+		if err := w.walk(&config); err != nil {
 			return nil, err
 		}
 	}

comp/core/secrets/secretsimpl/secrets_mock.go

@@ -16,9 +16,15 @@ import (
 	"go.uber.org/fx"
 )
 
+type callbackArgs struct {
+	yamlPath []string
+	value    any
+}
+
 // MockSecretResolver is a mock of the secret Component useful for testing
 type MockSecretResolver struct {
-	resolve map[string]string
+	resolve   map[string]string
+	callbacks []callbackArgs
 }
 
 var _ secrets.Component = (*MockSecretResolver)(nil)
@@ -29,13 +35,19 @@ func (m *MockSecretResolver) Configure(_ string, _ []string, _, _ int, _, _ bool
 // GetDebugInfo is not implemented
 func (m *MockSecretResolver) GetDebugInfo(_ io.Writer) {}
 
-// Inject adds data to be decrypted, by returning the value for the given key
+// Inject adds data to be resolved, by returning the value for the given key
 func (m *MockSecretResolver) Inject(key, value string) {
 	m.resolve[key] = value
 }
 
-// Decrypt returns the secret value based upon the injected data
-func (m *MockSecretResolver) Decrypt(data []byte, _ string) ([]byte, error) {
+// InjectCallback adds to the list of callbacks that will be used to mock ResolveWithCallback. Each injected callback
+// will equal a call to the callback givent to 'ResolveWithCallback'
+func (m *MockSecretResolver) InjectCallback(yamlPath []string, value any) {
+	m.callbacks = append(m.callbacks, callbackArgs{yamlPath: yamlPath, value: value})
+}
+
+// Resolve returns the secret value based upon the injected data
+func (m *MockSecretResolver) Resolve(data []byte, _ string) ([]byte, error) {
 	re := regexp.MustCompile(`ENC\[(.*?)\]`)
 	result := re.ReplaceAllStringFunc(string(data), func(in string) string {
 		key := in[4 : len(in)-1]
@@ -44,6 +56,14 @@ func (m *MockSecretResolver) Decrypt(data []byte, _ string) ([]byte, error) {
 	return []byte(result), nil
 }
 
+// ResolveWithCallback mocks the ResolveWithCallback method of the secrets Component
+func (m *MockSecretResolver) ResolveWithCallback(_ []byte, _ string, cb secrets.ResolveCallback) error {
+	for _, call := range m.callbacks {
+		cb(call.yamlPath, call.value)
+	}
+	return nil
+}
+
 // NewMockSecretResolver constructs a MockSecretResolver
 func NewMockSecretResolver() *MockSecretResolver {
 	return &MockSecretResolver{resolve: make(map[string]string)}