diff --git a/bootstrap/secret/secret.go b/bootstrap/secret/secret.go index 828922a8..5ff764da 100644 --- a/bootstrap/secret/secret.go +++ b/bootstrap/secret/secret.go @@ -74,6 +74,18 @@ func NewSecretProvider( secureProvider.SetClient(secretClient) provider = secureProvider lc.Info("Created SecretClient") + + lc.Debugf("SecretsFile is '%s'", secretConfig.SecretsFile) + + if len(strings.TrimSpace(secretConfig.SecretsFile)) == 0 { + lc.Infof("SecretsFile not set, skipping seeding of service secrets.") + break + } + + err = secureProvider.LoadServiceSecrets(secretStoreConfig) + if err != nil { + return nil, err + } break } } @@ -107,6 +119,7 @@ func getSecretConfig(secretStoreInfo config.SecretStoreInfo, tokenLoader authtok Host: secretStoreInfo.Host, Port: secretStoreInfo.Port, Path: addEdgeXSecretPathPrefix(secretStoreInfo.Path), + SecretsFile: secretStoreInfo.SecretsFile, Protocol: secretStoreInfo.Protocol, Namespace: secretStoreInfo.Namespace, RootCaCertPath: secretStoreInfo.RootCaCertPath, diff --git a/bootstrap/secret/secure.go b/bootstrap/secret/secure.go index 84c0a69b..bee0b184 100644 --- a/bootstrap/secret/secure.go +++ b/bootstrap/secret/secure.go @@ -18,12 +18,19 @@ package secret import ( "errors" "fmt" + "os" + "strings" "sync" "time" + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" + "github.com/hashicorp/go-multierror" + "github.com/edgexfoundry/go-mod-bootstrap/v2/bootstrap/interfaces" + "github.com/edgexfoundry/go-mod-bootstrap/v2/config" "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" + "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/authtokenloader" "github.com/edgexfoundry/go-mod-secrets/v2/secrets" ) @@ -169,7 +176,7 @@ func (p *SecureProvider) GetAccessToken(tokenType string, serviceKey string) (st } } -// defaultTokenExpiredCallback is the default implementation of tokenExpiredCallback function +// DefaultTokenExpiredCallback is the default implementation of tokenExpiredCallback function // It utilizes the tokenFile to re-read the token and enable retry if any update from the expired token func (p *SecureProvider) DefaultTokenExpiredCallback(expiredToken string) (replacementToken string, retry bool) { tokenFile := p.configuration.GetBootstrap().SecretStore.TokenFile @@ -190,3 +197,78 @@ func (p *SecureProvider) DefaultTokenExpiredCallback(expiredToken string) (repla return reReadToken, true } + +// LoadServiceSecrets loads the service secrets from the specified file and stores them in the service's SecretStore +func (p *SecureProvider) LoadServiceSecrets(secretStoreConfig config.SecretStoreInfo) error { + + contents, err := os.ReadFile(secretStoreConfig.SecretsFile) + if err != nil { + return fmt.Errorf("seeding secrets failed: %s", err.Error()) + } + + data, seedingErrs := p.seedSecrets(contents) + + if secretStoreConfig.DisableScrubSecretsFile { + p.lc.Infof("Scrubbing of secrets file disable.") + return seedingErrs + } + + if err := os.WriteFile(secretStoreConfig.SecretsFile, data, 0); err != nil { + return fmt.Errorf("seeding secrets failed: unable to overwrite file with secret data removed: %s", err.Error()) + } + + p.lc.Infof("Scrubbing of secrets file complete.") + + return seedingErrs +} + +func (p *SecureProvider) seedSecrets(contents []byte) ([]byte, error) { + serviceSecrets, err := UnmarshalServiceSecretsJson(contents) + if err != nil { + return nil, fmt.Errorf("seeding secrets failed unmarshaling JSON: %s", err.Error()) + } + + p.lc.Infof("Seeding %d Service Secrets", len(serviceSecrets.Secrets)) + + var seedingErrs error + for index, secret := range serviceSecrets.Secrets { + if secret.Imported { + p.lc.Infof("Secret for '%s' already imported. Skipping...", secret.Path) + continue + } + + // At this pint the JSON validation and above check cover all the required validation, so go to store secret. + path, data := prepareSecret(secret) + err := p.StoreSecret(path, data) + if err != nil { + message := fmt.Sprintf("failed to store secret for '%s': %s", secret.Path, err.Error()) + p.lc.Errorf(message) + seedingErrs = multierror.Append(seedingErrs, errors.New(message)) + continue + } + + p.lc.Infof("Secret for '%s' successfully stored.", secret.Path) + + serviceSecrets.Secrets[index].Imported = true + serviceSecrets.Secrets[index].SecretData = make([]common.SecretDataKeyValue, 0) + } + + // Now need to write the file back over with the imported secrets' secretData removed. + data, err := serviceSecrets.MarshalJson() + if err != nil { + return nil, fmt.Errorf("seeding secrets failed marshaling back to JSON to clear secrets: %s", err.Error()) + } + + return data, seedingErrs +} + +func prepareSecret(secret ServiceSecret) (string, map[string]string) { + var secretsKV = make(map[string]string) + for _, secret := range secret.SecretData { + secretsKV[secret.Key] = secret.Value + } + + path := strings.TrimSpace(secret.Path) + + return path, secretsKV +} diff --git a/bootstrap/secret/secure_test.go b/bootstrap/secret/secure_test.go index eb72872a..35237ea8 100644 --- a/bootstrap/secret/secure_test.go +++ b/bootstrap/secret/secure_test.go @@ -19,9 +19,12 @@ import ( "testing" "time" + mock2 "github.com/stretchr/testify/mock" + bootstrapConfig "github.com/edgexfoundry/go-mod-bootstrap/v2/config" "github.com/edgexfoundry/go-mod-core-contracts/v2/clients/logger" + "github.com/edgexfoundry/go-mod-secrets/v2/pkg" mocks2 "github.com/edgexfoundry/go-mod-secrets/v2/pkg/token/authtokenloader/mocks" "github.com/edgexfoundry/go-mod-secrets/v2/secrets" @@ -251,3 +254,49 @@ func TestSecureProvider_GetAccessToken(t *testing.T) { }) } } + +func TestSecureProvider_seedSecrets(t *testing.T) { + allGood := `{"secrets": [{"path": "auth","imported": false,"secretData": [{"key": "user1","value": "password1"}]}]}` + allGoodExpected := `{"secrets":[{"path":"auth","imported":true,"secretData":[]}]}` + badJson := `{"secrets": [{"path": "","imported": false,"secretData": null}]}` + + tests := []struct { + name string + secretsJson string + expectedJson string + mockError bool + expectedError string + }{ + {"Valid", allGood, allGoodExpected, false, ""}, + {"Partial Valid", allGood, allGoodExpected, false, ""}, + {"Bad JSON", badJson, "", false, "seeding secrets failed unmarshaling JSON: ServiceSecrets.Secrets[0].Path field should not be empty string; ServiceSecrets.Secrets[0].SecretData field is required"}, + {"Store Error", allGood, "", true, "1 error occurred:\n\t* failed to store secret for 'auth': store failed\n\n"}, + } + + target := NewSecureProvider(TestConfig{}, logger.MockLogger{}, nil) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + mock := &mocks.SecretClient{} + + if test.mockError { + mock.On("StoreSecrets", mock2.Anything, mock2.Anything).Return(errors.New("store failed")).Once() + } else { + mock.On("StoreSecrets", mock2.Anything, mock2.Anything).Return(nil).Once() + } + + target.SetClient(mock) + + actual, err := target.seedSecrets([]byte(test.secretsJson)) + if len(test.expectedError) > 0 { + require.Error(t, err) + assert.EqualError(t, err, test.expectedError) + return + } + + require.NoError(t, err) + assert.Equal(t, test.expectedJson, string(actual)) + }) + } +} diff --git a/bootstrap/secret/types.go b/bootstrap/secret/types.go new file mode 100644 index 00000000..23a1932e --- /dev/null +++ b/bootstrap/secret/types.go @@ -0,0 +1,69 @@ +/******************************************************************************* + * Copyright 2021 Intel Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + *******************************************************************************/ + +package secret + +import ( + "encoding/json" + "fmt" + + validation "github.com/edgexfoundry/go-mod-core-contracts/v2/common" + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" + "github.com/hashicorp/go-multierror" +) + +// ServiceSecrets contains the list of secrets to import into a service's SecretStore +type ServiceSecrets struct { + Secrets []ServiceSecret `json:"secrets" validate:"required,gt=0,dive"` +} + +// ServiceSecret contains the information about a service's secret to import into a service's SecretStore +type ServiceSecret struct { + Path string `json:"path" validate:"edgex-dto-none-empty-string"` + Imported bool `json:"imported"` + SecretData []common.SecretDataKeyValue `json:"secretData" validate:"required,dive"` +} + +// MarshalJson marshal the service's secrets to JSON. +func (s *ServiceSecrets) MarshalJson() ([]byte, error) { + return json.Marshal(s) +} + +// UnmarshalServiceSecretsJson un-marshals the JSON containing the services list of secrets +func UnmarshalServiceSecretsJson(data []byte) (*ServiceSecrets, error) { + secrets := &ServiceSecrets{} + + if err := json.Unmarshal(data, secrets); err != nil { + return nil, err + } + + if err := validation.Validate(secrets); err != nil { + return nil, err + } + + var validationErrs error + + // Since secretData len validation can't be specified to only validate when Imported=false, we have to do it manually here + for _, secret := range secrets.Secrets { + if !secret.Imported && len(secret.SecretData) == 0 { + validationErrs = multierror.Append(validationErrs, fmt.Errorf("SecretData for '%s' must not be empty when Imported=false", secret.Path)) + } + } + + if validationErrs != nil { + return nil, validationErrs + } + + return secrets, nil +} diff --git a/bootstrap/secret/types_test.go b/bootstrap/secret/types_test.go new file mode 100644 index 00000000..ef290a5b --- /dev/null +++ b/bootstrap/secret/types_test.go @@ -0,0 +1,173 @@ +/******************************************************************************* + * Copyright 2021 Intel Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + *******************************************************************************/ + +package secret + +import ( + "testing" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServiceSecrets_UnmarshalJson_Imported_false(t *testing.T) { + expected := ServiceSecrets{ + []ServiceSecret{ + { + Path: "credentials001", + Imported: false, + SecretData: []common.SecretDataKeyValue{{ + Key: "user1", + Value: "password1", + }}, + }, + { + Path: "credentials002", + Imported: false, + SecretData: []common.SecretDataKeyValue{{ + Key: "user2", + Value: "password2", + }}, + }, + }, + } + + data := ` { + "secrets": [ + { + "path": "credentials001", + "imported": false, + "secretData": [ + { + "key": "user1", + "value": "password1" + } + ] + }, + { + "path": "credentials002", + "imported": false, + "secretData": [ + { + "key": "user2", + "value": "password2" + } + ] + } + ] +} +` + + secrets, err := UnmarshalServiceSecretsJson([]byte(data)) + require.NoError(t, err) + assert.Equal(t, expected, *secrets) +} + +func TestServiceSecrets_UnmarshalJson_Imported_true(t *testing.T) { + expected := ServiceSecrets{ + []ServiceSecret{ + { + Path: "credentials001", + Imported: true, + SecretData: make([]common.SecretDataKeyValue, 0), + }, + { + Path: "credentials002", + Imported: true, + SecretData: make([]common.SecretDataKeyValue, 0), + }, + }, + } + + data := ` { + "secrets": [ + { + "path": "credentials001", + "imported": true, + "secretData": [] + }, + { + "path": "credentials002", + "imported": true, + "secretData": [] + } + ] +} +` + + secrets, err := UnmarshalServiceSecretsJson([]byte(data)) + require.NoError(t, err) + assert.Equal(t, expected, *secrets) +} + +func TestServiceSecrets_UnmarshalJson_Failed_Validation(t *testing.T) { + allGood := `{"secrets": [{"path": "auth","imported": false,"secretData": [{"key": "user1","value": "password1"}]}]}` + noPath := `{"secrets": [{"path": "","imported": false,"secretData": [{"key": "user1","value": "password1"}]}]}` + noSecretData := `{"secrets": [{"path": "auth","imported": false}]}` + emptySecretData := `{"secrets": [{"path": "auth","imported": false, "secretData": []}]}` + missingKey := `{"secrets": [{"path": "auth","imported": false,"secretData": [{"value": "password1"}]}]}` + missingValue := `{"secrets": [{"path": "auth","imported": false,"secretData": [{"key": "user1"}]}]}` + + tests := []struct { + name string + data string + expectedError string + }{ + {"All good", allGood, ""}, + {"Empty JSON", `{}`, "ServiceSecrets.Secrets field is required"}, + {"No Secrets", `{"secrets": []}`, "ServiceSecrets.Secrets field should greater than 0"}, + {"No Path", noPath, "ServiceSecrets.Secrets[0].Path field should not be empty string"}, + {"No SecretData", noSecretData, "ServiceSecrets.Secrets[0].SecretData field is required"}, + {"Empty SecretData", emptySecretData, "1 error occurred:\n\t* SecretData for 'auth' must not be empty when Imported=false\n\n"}, + {"Missing Key", missingKey, "ServiceSecrets.Secrets[0].SecretData[0].Key field is required"}, + {"Missing Value", missingValue, "ServiceSecrets.Secrets[0].SecretData[0].Value field is required"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := UnmarshalServiceSecretsJson([]byte(test.data)) + if len(test.expectedError) == 0 { + require.NoError(t, err) + return + } + + require.Error(t, err) + assert.EqualError(t, err, test.expectedError) + }) + } + +} + +func TestServiceSecrets_MarshalJson(t *testing.T) { + expected := `{"secrets":[{"path":"credentials001","imported":true,"secretData":[]},{"path":"credentials002","imported":true,"secretData":[]}]}` + secrets := ServiceSecrets{ + []ServiceSecret{ + { + Path: "credentials001", + Imported: true, + SecretData: make([]common.SecretDataKeyValue, 0), + }, + { + Path: "credentials002", + SecretData: make([]common.SecretDataKeyValue, 0), + Imported: true, + }, + }, + } + + data, err := secrets.MarshalJson() + require.NoError(t, err) + assert.Equal(t, expected, string(data)) +} diff --git a/config/types.go b/config/types.go index e2de09d9..5837349f 100644 --- a/config/types.go +++ b/config/types.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/edgexfoundry/go-mod-core-contracts/v2/common" + "github.com/edgexfoundry/go-mod-secrets/v2/pkg/types" ) @@ -107,6 +108,11 @@ type SecretStoreInfo struct { Authentication types.AuthenticationInfo // TokenFile provides a location to a token file. TokenFile string + // SecretsFile is optional Path to JSON file containing secrets to seed into service's SecretStore + SecretsFile string + // DisableScrubSecretsFile specifies to not scrub secrets file after importing. Service will fail start-up if + // not disabled and file can not be written. + DisableScrubSecretsFile bool } type Database struct { diff --git a/go.mod b/go.mod index 3a0d556f..90245ed3 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,9 @@ require ( github.com/edgexfoundry/go-mod-configuration/v2 v2.0.0 github.com/edgexfoundry/go-mod-core-contracts/v2 v2.0.0 github.com/edgexfoundry/go-mod-registry/v2 v2.0.0 - github.com/edgexfoundry/go-mod-secrets/v2 v2.0.0 + github.com/edgexfoundry/go-mod-secrets/v2 v2.0.1-dev.4 github.com/gorilla/mux v1.7.4 + github.com/hashicorp/go-multierror v1.1.0 github.com/pelletier/go-toml v1.9.4 github.com/stretchr/testify v1.7.0 )