feat: Add optional capability to seed service secrets (#276)

* feat: Add optional capability to seed service secrets

Secrets are seeded from a JSON file specified by the SecretStore.SecretsFile setting
If SecretsFile setting is blank, seeding is skipped.

* feat: Add DisableScrubSecretsFile setting to control scrubbing of secrets file

closes #273

Signed-off-by: Leonard Goodell <[email protected]>

bootstrap/secret/secret.go

@@ -74,6 +74,18 @@ func NewSecretProvider(
 					secureProvider.SetClient(secretClient)
 					provider = secureProvider
 					lc.Info("Created SecretClient")
+
+					lc.Debugf("SecretsFile is '%s'", secretConfig.SecretsFile)
+
+					if len(strings.TrimSpace(secretConfig.SecretsFile)) == 0 {
+						lc.Infof("SecretsFile not set, skipping seeding of service secrets.")
+						break
+					}
+
+					err = secureProvider.LoadServiceSecrets(secretStoreConfig)
+					if err != nil {
+						return nil, err
+					}
 					break
 				}
 			}
@@ -107,6 +119,7 @@ func getSecretConfig(secretStoreInfo config.SecretStoreInfo, tokenLoader authtok
 		Host:           secretStoreInfo.Host,
 		Port:           secretStoreInfo.Port,
 		Path:           addEdgeXSecretPathPrefix(secretStoreInfo.Path),
+		SecretsFile:    secretStoreInfo.SecretsFile,
 		Protocol:       secretStoreInfo.Protocol,
 		Namespace:      secretStoreInfo.Namespace,
 		RootCaCertPath: secretStoreInfo.RootCaCertPath,

bootstrap/secret/secure.go

@@ -18,12 +18,19 @@ package secret
 import (
 	"errors"
 	"fmt"
+	"os"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common"
+	"github.com/hashicorp/go-multierror"
+
 	"github.com/edgexfoundry/go-mod-bootstrap/v2/bootstrap/interfaces"
+	"github.com/edgexfoundry/go-mod-bootstrap/v2/config"
 
 	"github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger"
+
 	"github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/authtokenloader"
 	"github.com/edgexfoundry/go-mod-secrets/v2/secrets"
 )
@@ -169,7 +176,7 @@ func (p *SecureProvider) GetAccessToken(tokenType string, serviceKey string) (st
 	}
 }
 
-// defaultTokenExpiredCallback is the default implementation of tokenExpiredCallback function
+// DefaultTokenExpiredCallback is the default implementation of tokenExpiredCallback function
 // It utilizes the tokenFile to re-read the token and enable retry if any update from the expired token
 func (p *SecureProvider) DefaultTokenExpiredCallback(expiredToken string) (replacementToken string, retry bool) {
 	tokenFile := p.configuration.GetBootstrap().SecretStore.TokenFile
@@ -190,3 +197,78 @@ func (p *SecureProvider) DefaultTokenExpiredCallback(expiredToken string) (repla
 
 	return reReadToken, true
 }
+
+// LoadServiceSecrets loads the service secrets from the specified file and stores them in the service's SecretStore
+func (p *SecureProvider) LoadServiceSecrets(secretStoreConfig config.SecretStoreInfo) error {
+
+	contents, err := os.ReadFile(secretStoreConfig.SecretsFile)
+	if err != nil {
+		return fmt.Errorf("seeding secrets failed: %s", err.Error())
+	}
+
+	data, seedingErrs := p.seedSecrets(contents)
+
+	if secretStoreConfig.DisableScrubSecretsFile {
+		p.lc.Infof("Scrubbing of secrets file disable.")
+		return seedingErrs
+	}
+
+	if err := os.WriteFile(secretStoreConfig.SecretsFile, data, 0); err != nil {
+		return fmt.Errorf("seeding secrets failed: unable to overwrite file with secret data removed: %s", err.Error())
+	}
+
+	p.lc.Infof("Scrubbing of secrets file complete.")
+
+	return seedingErrs
+}
+
+func (p *SecureProvider) seedSecrets(contents []byte) ([]byte, error) {
+	serviceSecrets, err := UnmarshalServiceSecretsJson(contents)
+	if err != nil {
+		return nil, fmt.Errorf("seeding secrets failed unmarshaling JSON: %s", err.Error())
+	}
+
+	p.lc.Infof("Seeding %d Service Secrets", len(serviceSecrets.Secrets))
+
+	var seedingErrs error
+	for index, secret := range serviceSecrets.Secrets {
+		if secret.Imported {
+			p.lc.Infof("Secret for '%s' already imported. Skipping...", secret.Path)
+			continue
+		}
+
+		// At this pint the JSON validation and above check cover all the required validation, so go to store secret.
+		path, data := prepareSecret(secret)
+		err := p.StoreSecret(path, data)
+		if err != nil {
+			message := fmt.Sprintf("failed to store secret for '%s': %s", secret.Path, err.Error())
+			p.lc.Errorf(message)
+			seedingErrs = multierror.Append(seedingErrs, errors.New(message))
+			continue
+		}
+
+		p.lc.Infof("Secret for '%s' successfully stored.", secret.Path)
+
+		serviceSecrets.Secrets[index].Imported = true
+		serviceSecrets.Secrets[index].SecretData = make([]common.SecretDataKeyValue, 0)
+	}
+
+	// Now need to write the file back over with the imported secrets' secretData removed.
+	data, err := serviceSecrets.MarshalJson()
+	if err != nil {
+		return nil, fmt.Errorf("seeding secrets failed marshaling back to JSON to clear secrets: %s", err.Error())
+	}
+
+	return data, seedingErrs
+}
+
+func prepareSecret(secret ServiceSecret) (string, map[string]string) {
+	var secretsKV = make(map[string]string)
+	for _, secret := range secret.SecretData {
+		secretsKV[secret.Key] = secret.Value
+	}
+
+	path := strings.TrimSpace(secret.Path)
+
+	return path, secretsKV
+}

bootstrap/secret/secure_test.go

@@ -19,9 +19,12 @@ import (
 	"testing"
 	"time"
 
+	mock2 "github.com/stretchr/testify/mock"
+
 	bootstrapConfig "github.com/edgexfoundry/go-mod-bootstrap/v2/config"
 
 	"github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger"
+
 	"github.com/edgexfoundry/go-mod-secrets/v2/pkg"
 	mocks2 "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/authtokenloader/mocks"
 	"github.com/edgexfoundry/go-mod-secrets/v2/secrets"
@@ -251,3 +254,49 @@ func TestSecureProvider_GetAccessToken(t *testing.T) {
 		})
 	}
 }
+
+func TestSecureProvider_seedSecrets(t *testing.T) {
+	allGood := `{"secrets": [{"path": "auth","imported": false,"secretData": [{"key": "user1","value": "password1"}]}]}`
+	allGoodExpected := `{"secrets":[{"path":"auth","imported":true,"secretData":[]}]}`
+	badJson := `{"secrets": [{"path": "","imported": false,"secretData": null}]}`
+
+	tests := []struct {
+		name          string
+		secretsJson   string
+		expectedJson  string
+		mockError     bool
+		expectedError string
+	}{
+		{"Valid", allGood, allGoodExpected, false, ""},
+		{"Partial Valid", allGood, allGoodExpected, false, ""},
+		{"Bad JSON", badJson, "", false, "seeding secrets failed unmarshaling JSON: ServiceSecrets.Secrets[0].Path field should not be empty string; ServiceSecrets.Secrets[0].SecretData field is required"},
+		{"Store Error", allGood, "", true, "1 error occurred:\n\t* failed to store secret for 'auth': store failed\n\n"},
+	}
+
+	target := NewSecureProvider(TestConfig{}, logger.MockLogger{}, nil)
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			mock := &mocks.SecretClient{}
+
+			if test.mockError {
+				mock.On("StoreSecrets", mock2.Anything, mock2.Anything).Return(errors.New("store failed")).Once()
+			} else {
+				mock.On("StoreSecrets", mock2.Anything, mock2.Anything).Return(nil).Once()
+			}
+
+			target.SetClient(mock)
+
+			actual, err := target.seedSecrets([]byte(test.secretsJson))
+			if len(test.expectedError) > 0 {
+				require.Error(t, err)
+				assert.EqualError(t, err, test.expectedError)
+				return
+			}
+
+			require.NoError(t, err)
+			assert.Equal(t, test.expectedJson, string(actual))
+		})
+	}
+}

bootstrap/secret/types.go

@@ -0,0 +1,69 @@
+/*******************************************************************************
+ * Copyright 2021 Intel Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *******************************************************************************/
+
+package secret
+
+import (
+	"encoding/json"
+	"fmt"
+
+	validation "github.com/edgexfoundry/go-mod-core-contracts/v2/common"
+	"github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common"
+	"github.com/hashicorp/go-multierror"
+)
+
+// ServiceSecrets contains the list of secrets to import into a service's SecretStore
+type ServiceSecrets struct {
+	Secrets []ServiceSecret `json:"secrets" validate:"required,gt=0,dive"`
+}
+
+// ServiceSecret contains the information about a service's secret to import into a service's SecretStore
+type ServiceSecret struct {
+	Path       string                      `json:"path" validate:"edgex-dto-none-empty-string"`
+	Imported   bool                        `json:"imported"`
+	SecretData []common.SecretDataKeyValue `json:"secretData" validate:"required,dive"`
+}
+
+// MarshalJson marshal the service's secrets to JSON.
+func (s *ServiceSecrets) MarshalJson() ([]byte, error) {
+	return json.Marshal(s)
+}
+
+// UnmarshalServiceSecretsJson un-marshals the JSON containing the services list of secrets
+func UnmarshalServiceSecretsJson(data []byte) (*ServiceSecrets, error) {
+	secrets := &ServiceSecrets{}
+
+	if err := json.Unmarshal(data, secrets); err != nil {
+		return nil, err
+	}
+
+	if err := validation.Validate(secrets); err != nil {
+		return nil, err
+	}
+
+	var validationErrs error
+
+	// Since secretData len validation can't be specified to only validate when Imported=false, we have to do it manually here
+	for _, secret := range secrets.Secrets {
+		if !secret.Imported && len(secret.SecretData) == 0 {
+			validationErrs = multierror.Append(validationErrs, fmt.Errorf("SecretData for '%s' must not be empty when Imported=false", secret.Path))
+		}
+	}
+
+	if validationErrs != nil {
+		return nil, validationErrs
+	}
+
+	return secrets, nil
+}