init.go

/*******************************************************************************
 * Copyright 2022-2023 Intel Corporation
 * Copyright 2019 Dell Inc.
 * Copyright 2024 IOTech Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 *
 *******************************************************************************/

package secretstore

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/edgexfoundry/edgex-go/internal"
	"github.com/edgexfoundry/edgex-go/internal/security/kdf"
	"github.com/edgexfoundry/edgex-go/internal/security/pipedhexreader"
	"github.com/edgexfoundry/edgex-go/internal/security/secretstore/config"
	"github.com/edgexfoundry/edgex-go/internal/security/secretstore/container"
	"github.com/edgexfoundry/edgex-go/internal/security/secretstore/secretsengine"
	"github.com/edgexfoundry/edgex-go/internal/security/secretstore/tokenfilewriter"

	bootstrapContainer "github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/container"
	"github.com/edgexfoundry/go-mod-bootstrap/v4/bootstrap/startup"
	"github.com/edgexfoundry/go-mod-bootstrap/v4/di"

	"github.com/edgexfoundry/go-mod-core-contracts/v4/clients/logger"
	"github.com/edgexfoundry/go-mod-core-contracts/v4/common"

	"github.com/edgexfoundry/go-mod-secrets/v4/pkg"
	"github.com/edgexfoundry/go-mod-secrets/v4/pkg/token/fileioperformer"
	"github.com/edgexfoundry/go-mod-secrets/v4/pkg/types"
	"github.com/edgexfoundry/go-mod-secrets/v4/secrets"
)

const (
	addKnownSecretsEnv   = "EDGEX_ADD_KNOWN_SECRETS" // nolint:gosec
	redisSecretName      = "redisdb"
	postgresSecretName   = "postgres"
	messagebusSecretName = "message-bus"
	knownSecretSeparator = ","
	serviceListBegin     = "["
	serviceListEnd       = "]"
	serviceListSeparator = ";"
	secretBasePath       = "/v1/secret/edgex" // nolint:gosec
	defaultMsgBusUser    = "msgbususer"
)

var errNotFound = errors.New("credential NOT found")

type Bootstrap struct {
	insecureSkipVerify  bool
	secretStoreInterval int
	validKnownSecrets   map[string]bool
}

func NewBootstrap(insecureSkipVerify bool, secretStoreInterval int) *Bootstrap {
	return &Bootstrap{
		insecureSkipVerify:  insecureSkipVerify,
		secretStoreInterval: secretStoreInterval,
		validKnownSecrets:   map[string]bool{redisSecretName: true, messagebusSecretName: true},
	}
}

// BootstrapHandler fulfills the BootstrapHandler contract and performs initialization needed by the data service.
func (b *Bootstrap) BootstrapHandler(ctx context.Context, _ *sync.WaitGroup, _ startup.Timer, dic *di.Container) bool {
	configuration := container.ConfigurationFrom(dic.Get)
	secretStoreConfig := configuration.SecretStore
	lc := bootstrapContainer.LoggingClientFrom(dic.Get)

	//step 1: boot up secretstore general steps same as other EdgeX microservice

	//step 2: initialize the communications
	fileOpener := fileioperformer.NewDefaultFileIoPerformer()

	var httpCaller internal.HttpCaller
	if caFilePath := secretStoreConfig.CaFilePath; caFilePath != "" {
		lc.Info("using certificate verification for secret store connection")
		caReader, err := fileOpener.OpenFileReader(caFilePath, os.O_RDONLY, 0400)
		if err != nil {
			lc.Errorf("failed to load CA certificate: %s", err.Error())
			return false
		}
		httpCaller = pkg.NewRequester(lc).WithTLS(caReader, secretStoreConfig.ServerName)
	} else {
		lc.Info("bypassing certificate verification for secret store connection")
		httpCaller = pkg.NewRequester(lc).Insecure()
	}

	intervalDuration := time.Duration(b.secretStoreInterval) * time.Second
	clientConfig := types.SecretConfig{
		Type:     secretStoreConfig.Type,
		Protocol: secretStoreConfig.Protocol,
		Host:     secretStoreConfig.Host,
		Port:     secretStoreConfig.Port,
	}
	client, err := secrets.NewSecretStoreClient(clientConfig, lc, httpCaller)
	if err != nil {
		lc.Errorf("failed to create SecretStoreClient: %s", err.Error())
		return false
	}

	lc.Info("SecretStoreClient created")

	pipedHexReader := pipedhexreader.NewPipedHexReader()
	keyDeriver := kdf.NewKdf(fileOpener, secretStoreConfig.TokenFolderPath, sha256.New)
	vmkEncryption := NewVMKEncryption(fileOpener, pipedHexReader, keyDeriver)

	hook := os.Getenv("EDGEX_IKM_HOOK")
	if len(hook) > 0 {
		err := vmkEncryption.LoadIKM(hook)
		defer vmkEncryption.WipeIKM() // Ensure IKM is wiped from memory
		if err != nil {
			lc.Errorf("failed to setup %s master key encryption: %s", secrets.DefaultSecretStore, err.Error())
			return false
		}
		lc.Infof("Enabled encryption of %s master key", secrets.DefaultSecretStore)
	} else {
		lc.Infof("%s master key encryption not enabled. EDGEX_IKM_HOOK not set.", secrets.DefaultSecretStore)
	}

	var initResponse types.InitResponse // reused many places in below flow

	//step 3: initialize and unseal secret store
	for shouldContinue := true; shouldContinue; {
		// Anonymous function used to prevent file handles from accumulating
		terminalFailure := func() bool {
			sCode, _ := client.HealthCheck()

			switch sCode {
			case http.StatusOK:
				// Load the init response from disk since we need it to regenerate root token later
				if err := LoadInitResponse(lc, fileOpener, secretStoreConfig, &initResponse); err != nil {
					lc.Errorf("unable to load init response: %s", err.Error())
					return true
				}
				lc.Infof("%s is initialized and unsealed (status code: %d)", secrets.DefaultSecretStore, sCode)
				shouldContinue = false

			case http.StatusTooManyRequests:
				// we're done here. Will go into ready mode or reseal
				shouldContinue = false

			case http.StatusNotImplemented:
				lc.Infof("%s is not initialized (status code: %d). Starting initialization and unseal phases", secrets.DefaultSecretStore, sCode)
				initResponse, err = client.Init(secretStoreConfig.SecretThreshold, secretStoreConfig.SecretShares)
				if err != nil {
					lc.Errorf("Unable to Initialize %s: %s. Will try again...", secrets.DefaultSecretStore, err.Error())
					// Not terminal failure, should continue and try again
					return false
				}

				if secretStoreConfig.RevokeRootTokens {
					// Never persist the root token to disk on secret store initialization if we intend to revoke it later
					initResponse.RootToken = ""
					lc.Info("Root token stripped from init response for security reasons")
				}

				err = client.Unseal(initResponse.KeysBase64)
				if err != nil {
					lc.Errorf("Unable to unseal %s: %s", secrets.DefaultSecretStore, err.Error())
					return true
				}

				// We need the unencrypted initResponse in order to generate a temporary root token later
				// Make a copy and save the copy, possibly encrypted
				encryptedInitResponse := initResponse
				// Optionally encrypt the secret store init response based on whether encryption was enabled
				if vmkEncryption.IsEncrypting() {
					if err := vmkEncryption.EncryptInitResponse(&encryptedInitResponse); err != nil {
						lc.Errorf("failed to encrypt init response from secret store: %s", err.Error())
						return true
					}
				}
				if err := saveInitResponse(lc, fileOpener, secretStoreConfig, &encryptedInitResponse); err != nil {
					lc.Errorf("unable to save init response: %s", err.Error())
					return true
				}

			case http.StatusServiceUnavailable:
				lc.Infof("%s is sealed (status code: %d). Starting unseal phase", secrets.DefaultSecretStore, sCode)
				if err := LoadInitResponse(lc, fileOpener, secretStoreConfig, &initResponse); err != nil {
					lc.Errorf("unable to load init response: %s", err.Error())
					return true
				}
				// Optionally decrypt the secret store init response based on whether encryption was enabled
				if vmkEncryption.IsEncrypting() {
					err = vmkEncryption.DecryptInitResponse(&initResponse)
					if err != nil {
						lc.Errorf("failed to decrypt key shares for secret store unsealing: %s", err.Error())
						return true
					}
				}

				err := client.Unseal(initResponse.KeysBase64)
				if err == nil {
					shouldContinue = false
				}

			default:
				if sCode == 0 {
					lc.Errorf("%s is in an unknown state. No Status code available", secrets.DefaultSecretStore)
				} else {
					lc.Errorf("%s is in an unknown state. Status code: %d", secrets.DefaultSecretStore, sCode)
				}
			}

			return false
		}()

		if terminalFailure {
			return false
		}

		if shouldContinue {
			lc.Infof("trying %s init/unseal again in %d seconds", secrets.DefaultSecretStore, b.secretStoreInterval)
			time.Sleep(intervalDuration)
		}
	}

	/* After secret store is initialized and unsealed, it takes a while to get ready to accept any request. During which period any request will get http 500 error.
	We need to check the status constantly until it return http StatusOK.
	*/
	ticker := time.NewTicker(time.Second)
	healthOkCh := make(chan struct{})
	go func() {
		for {
			<-ticker.C
			if sCode, _ := client.HealthCheck(); sCode == http.StatusOK {
				close(healthOkCh)
				ticker.Stop()
				return
			}
		}
	}()

	// Wait on a StatusOK response from client.HealthCheck()
	<-healthOkCh

	// create new root token
	// defer revoke token
	// optional: revoke other root token
	// revoke old tokens
	// create delegate credential
	// spawn token provider
	// create db credentials
	// upload kong certificate
	tokenMaintenance := NewTokenMaintenance(lc, client)

	// Create a transient root token from the key shares
	var rootToken string
	rootToken, err = client.RegenRootToken(initResponse.Keys)
	if err != nil {
		lc.Errorf("could not regenerate root token %s", err.Error())
		return false
	}
	defer func() {
		// Revoke transient root token at the end of this function
		lc.Info("revoking temporary root token")
		err := client.RevokeToken(rootToken)
		if err != nil {
			lc.Errorf("could not revoke temporary root token %s", err.Error())
		}
	}()
	lc.Info("generated transient root token")

	// Revoke the other root tokens
	if secretStoreConfig.RevokeRootTokens {
		if initResponse.RootToken != "" {
			initResponse.RootToken = ""
			if err := saveInitResponse(lc, fileOpener, secretStoreConfig, &initResponse); err != nil {
				lc.Errorf("unable to save init response: %s", err.Error())
				return false
			}
			lc.Info("Root token stripped from init response (on disk) for security reasons")
		}
		if err := tokenMaintenance.RevokeRootTokens(rootToken); err != nil {
			lc.Warnf("failed to revoke non-transient root tokens %s", err.Error())
		}
		lc.Info("completed cleanup of old root tokens")
	} else {
		lc.Info("not revoking existing root tokens")
	}

	// Revoke non-root tokens from previous runs
	if err := tokenMaintenance.RevokeNonRootTokens(rootToken); err != nil {
		lc.Warn("failed to revoke non-root tokens")
	}
	lc.Info("completed cleanup of old admin/service tokens")

	// If configured to do so, create a token issuing token
	if secretStoreConfig.TokenProviderAdminTokenPath != "" {
		revokeIssuingTokenFuc, err := tokenfilewriter.NewWriter(lc, client, fileOpener).
			CreateAndWrite(rootToken, secretStoreConfig.TokenProviderAdminTokenPath, tokenMaintenance.CreateTokenIssuingToken)
		if err != nil {
			lc.Errorf("failed to create token issuing token: %s", err.Error())
			return false
		}
		if secretStoreConfig.TokenProviderType == OneShotProvider {
			// Revoke the admin token at the end of the current function if running a one-shot provider
			// otherwise assume the token provider will keep its token fresh after this point
			defer revokeIssuingTokenFuc()
		}
	}

	// Enable userpass auth engine
	upAuthEnabled, err := client.CheckAuthMethodEnabled(rootToken, UPAuthMountPoint, UserPassAuthEngine)
	if err != nil {
		lc.Errorf("failed to check if %s auth method enabled: %s", UserPassAuthEngine, err.Error())
		return false
	} else if !upAuthEnabled {
		// Enable userpass engine at /v1/auth/{eng.path} path (/v1 prefix supplied by secret store)
		lc.Infof("Enabling userpass authentication for the first time...")
		if err := client.EnablePasswordAuth(rootToken, UPAuthMountPoint); err != nil {
			lc.Errorf("failed to enable userpass secrets engine: %s", err.Error())
			return false
		}
		lc.Infof("Userpass authentication engine enabled at path %s", UPAuthMountPoint)
	}

	// Create a key for issuing JWTs
	keyExists, err := client.CheckIdentityKeyExists(rootToken, "edgex-identity")
	if err != nil {
		lc.Errorf("failed to check for JWT issuing key: %s", err.Error())
		return false
	} else if !keyExists {
		if err := client.CreateNamedIdentityKey(rootToken, "edgex-identity", "ES384"); err != nil {
			lc.Errorf("failed to create JWT issuing key: %s", err.Error())
			return false
		}
	}

	//Step 4: Launch token handler
	tokenProvider := NewTokenProvider(ctx, lc, NewDefaultExecRunner())
	if secretStoreConfig.TokenProvider != "" {
		if err := tokenProvider.SetConfiguration(secretStoreConfig); err != nil {
			lc.Errorf("failed to configure token provider: %s", err.Error())
			return false
		}
		if err := tokenProvider.Launch(); err != nil {
			lc.Errorf("token provider failed: %s", err.Error())
			return false
		}
	} else {
		lc.Info("no token provider configured")
	}

	// Enable KV secret engine
	if err := secretsengine.New(secretsengine.KVSecretsEngineMountPoint, secretsengine.KeyValue).
		Enable(&rootToken, lc, client); err != nil {
		lc.Errorf("failed to enable KV secrets engine: %s", err.Error())
		return false
	}

	// set the validKnownSecrets to true based on the database type
	if configuration.GetDatabaseInfo().Type == postgresSecretName {
		b.validKnownSecrets[postgresSecretName] = true
	}
	knownSecretsToAdd, err := b.getKnownSecretsToAdd()
	if err != nil {
		lc.Error(err.Error())
		return false
	}

	// credential creation
	gen := NewPasswordGenerator(lc, secretStoreConfig.PasswordProvider, secretStoreConfig.PasswordProviderArgs)
	secretStore := NewCred(httpCaller, rootToken, gen, secretStoreConfig.GetBaseURL(), lc)

	// continue credential creation

	// A little note on why there are two secrets names. For each microservice, the redis
	// username/password is uploaded to the secret store on both /v1/secret/edgex/%s/redisdb and
	// /v1/secret/edgex/redisdb/%s). The go-mod-secrets client requires a SecretName property to prefix all
	// secrets.
	// So edgex/%s/redisdb is for the microservices (microservices are restricted to their specific
	// edgex/%s), and edgex/redisdb/* is enumerated to initialize the database.
	// Similary for secure message bus credential.

	// Redis 5.x only supports a single shared password. When Redis 6 is released, this can be updated
	// to a per service password.

	redisCredentials, err := getCredential("security-bootstrapper-redis", secretStore, redisSecretName)
	if err != nil {
		if err != errNotFound {
			lc.Errorf("failed to determine if Redis credentials already exist or not: %s", err.Error())
			return false
		}

		lc.Info("Generating new password for Redis DB")
		defaultPassword, err := secretStore.GeneratePassword(ctx)
		if err != nil {
			lc.Error("failed to generate default password for redisdb")
			return false
		}

		redisCredentials = UserPasswordPair{
			User:     "default",
			Password: defaultPassword,
		}
	} else {
		lc.Info("Redis DB credentials exist, skipping generating new password")
	}

	if configuration.GetDatabaseInfo().Type == postgresSecretName {
		err = genPostgresCredentials(dic, secretStore, knownSecretsToAdd, ctx)
		if err != nil {
			return false
		}
	}

	// NOTE: for now redis secrets will be generated for all database types
	// Add any additional services that need the known DB secret
	lc.Infof("adding any additional services using redisdb for knownSecrets...")
	services, ok := knownSecretsToAdd[redisSecretName]
	if ok {
		for _, service := range services {
			err = addServiceCredential(lc, redisSecretName, secretStore, service, redisCredentials)
			if err != nil {
				lc.Error(err.Error())
				return false
			}
		}
	}

	lc.Infof("adding redisdb secret name for internal services...")
	for _, info := range configuration.Databases {
		service := info.Service

		// add credentials to service path if specified and they're not already there
		if len(service) != 0 {
			err = addServiceCredential(lc, redisSecretName, secretStore, service, redisCredentials)
			if err != nil {
				lc.Error(err.Error())
				return false
			}
		}
	}
	// security-bootstrapper-redis uses the path /v1/secret/edgex/security-bootstrapper-redis/ and go-mod-bootstrap
	// with append the DB type (redisdb)
	err = storeCredential(lc, "security-bootstrapper-redis", secretStore, redisSecretName, redisCredentials)
	if err != nil {
		lc.Error(err.Error())
		return false
	}

	// for secure message bus creds
	var msgBusCredentials UserPasswordPair
	if configuration.SecureMessageBus.Type != redisSecureMessageBusType &&
		configuration.SecureMessageBus.Type != noneSecureMessageBusType &&
		configuration.SecureMessageBus.Type != blankSecureMessageBusType {
		msgBusCredentials, err = getCredential(internal.BootstrapMessageBusServiceKey, secretStore, messagebusSecretName)
		if err != nil {
			if err != errNotFound {
				lc.Errorf("failed to determine if %s credentials already exist or not: %s", configuration.SecureMessageBus.Type, err.Error())
				return false
			}

			lc.Infof("Generating new password for %s bus", configuration.SecureMessageBus.Type)
			msgBusPassword, err := secretStore.GeneratePassword(ctx)
			if err != nil {
				lc.Errorf("failed to generate password for %s bus", configuration.SecureMessageBus.Type)
				return false
			}

			msgBusCredentials = UserPasswordPair{
				User:     defaultMsgBusUser,
				Password: msgBusPassword,
			}
		} else {
			lc.Infof("%s bus credentials already exist, skipping generating new password", configuration.SecureMessageBus.Type)
		}

		lc.Infof("adding any additional services using %s for knownSecrets...", messagebusSecretName)
		services, ok := knownSecretsToAdd[messagebusSecretName]
		if ok {
			for _, service := range services {
				err = addServiceCredential(lc, messagebusSecretName, secretStore, service, msgBusCredentials)
				if err != nil {
					lc.Error(err.Error())
					return false
				}
			}
		}
		err = storeCredential(lc, internal.BootstrapMessageBusServiceKey, secretStore, messagebusSecretName, msgBusCredentials)
		if err != nil {
			lc.Error(err.Error())
			return false
		}
	}

	// determine the type of message bus
	messageBusType := configuration.SecureMessageBus.Type
	var creds UserPasswordPair
	supportedSecureType := true
	var secretName string
	switch messageBusType {
	case redisSecureMessageBusType:
		creds = redisCredentials
		secretName = redisSecretName
	case mqttSecureMessageBusType:
		creds = msgBusCredentials
		secretName = messagebusSecretName
	default:
		supportedSecureType = false
		lc.Warnf("secure message bus '%s' is not supported", messageBusType)
	}

	if supportedSecureType {
		lc.Infof("adding credentials for '%s' message bus for internal services...", messageBusType)
		for _, info := range configuration.SecureMessageBus.Services {
			service := info.Service

			// add credentials to service path if specified and they're not already there
			if len(service) != 0 {
				err = addServiceCredential(lc, secretName, secretStore, service, creds)
				if err != nil {
					lc.Error(err.Error())
					return false
				}
			}
		}
	}

	err = ConfigureSecureMessageBus(configuration.SecureMessageBus, creds, lc)
	if err != nil {
		lc.Errorf("failed to configure for Secure Message Bus: %s", err.Error())
		return false
	}

	// Concat all cert path secretStore values together to check for empty values
	certPathCheck := secretStoreConfig.CertPath +
		secretStoreConfig.CertFilePath +
		secretStoreConfig.KeyFilePath

	// If any of the previous three proxy cert path values are present (len > 0), attempt to upload to secret store
	if len(strings.TrimSpace(certPathCheck)) != 0 {

		// Grab the certificate & check to see if it's already in the secret store
		cert := NewCerts(httpCaller, secretStoreConfig.CertPath, rootToken, secretStoreConfig.GetBaseURL(), lc)
		existing, err := cert.AlreadyInStore()
		if err != nil {
			lc.Error(err.Error())
			return false
		}

		if existing {
			lc.Info("proxy certificate pair are in the secret store already, skip uploading")
			return false
		}

		lc.Info("proxy certificate pair are not in the secret store yet, uploading them")
		cp, err := cert.ReadFrom(secretStoreConfig.CertFilePath, secretStoreConfig.KeyFilePath)
		if err != nil {
			lc.Error("failed to get certificate pair from volume")
			return false
		}

		lc.Info("proxy certificate pair are loaded from volume successfully, will upload to secret store")

		err = cert.UploadToStore(cp)
		if err != nil {
			lc.Error("failed to upload the proxy cert pair into the secret store")
			lc.Error(err.Error())
			return false
		}

		lc.Info("proxy certificate pair are uploaded to secret store successfully")

	} else {
		lc.Info("proxy certificate pair upload was skipped because cert secretStore value(s) were blank")
	}

	lc.Infof("%s init done successfully", secrets.DefaultSecretStore)
	return true

}

func (b *Bootstrap) getKnownSecretsToAdd() (map[string][]string, error) {
	// Process the env var for adding known secrets to the specified services' secret stores.
	// Format of the env var value is:
	//   "<secretName>[<serviceName>;<serviceName>; ...], <secretName>[<serviceName>;<serviceName>; ...], ..."
	knownSecretsToAdd := map[string][]string{}

	addKnownSecretsValue := strings.TrimSpace(os.Getenv(addKnownSecretsEnv))
	if len(addKnownSecretsValue) == 0 {
		return knownSecretsToAdd, nil
	}

	serviceNameRegx := regexp.MustCompile(ServiceNameValidationRegx)
	knownSecrets := strings.Split(addKnownSecretsValue, knownSecretSeparator)
	for _, secretSpec := range knownSecrets {
		// each secretSpec has format of "<secretName>[<serviceName>;<serviceName>; ...]"
		secretItems := strings.Split(secretSpec, serviceListBegin)
		if len(secretItems) != 2 {
			return nil, fmt.Errorf(
				"invalid specification for %s environment variable: Format of value '%s' is invalid. Missing or too many '%s'",
				addKnownSecretsEnv,
				secretSpec,
				serviceListBegin)
		}

		secretName := strings.TrimSpace(secretItems[0])

		_, valid := b.validKnownSecrets[secretName]
		if !valid {
			return nil, fmt.Errorf(
				"invalid specification for %s environment variable: '%s' is not a known secret",
				addKnownSecretsEnv,
				secretName)
		}

		serviceNameList := secretItems[1]
		if !strings.Contains(serviceNameList, serviceListEnd) {
			return nil, fmt.Errorf(
				"invalid specification for %s environment variable: Service list for '%s' missing closing '%s'",
				addKnownSecretsEnv,
				secretName,
				serviceListEnd)
		}

		serviceNameList = strings.TrimSpace(strings.Replace(serviceNameList, serviceListEnd, "", 1))
		if len(serviceNameList) == 0 {
			return nil, fmt.Errorf(
				"invalid specification for %s environment variable: Service name list for '%s' is empty.",
				addKnownSecretsEnv,
				secretName)
		}

		serviceNames := strings.Split(serviceNameList, serviceListSeparator)
		for index := range serviceNames {
			serviceNames[index] = strings.TrimSpace(serviceNames[index])

			if !serviceNameRegx.MatchString(serviceNames[index]) {
				return nil, fmt.Errorf(
					"invalid specification for %s environment variable: Service name '%s' has invalid characters.",
					addKnownSecretsEnv, serviceNames[index])
			}
		}

		// This supports listing known secret multiple times.
		// Same service name listed twice is not an issue since the add logic checks if the secret is already present.
		existingServices := knownSecretsToAdd[secretName]
		knownSecretsToAdd[secretName] = append(existingServices, serviceNames...)
	}

	return knownSecretsToAdd, nil
}

// XXX Collapse addServiceCredential and addDBCredential together by passing in the path or using
// variadic functions

func addServiceCredential(lc logger.LoggingClient, secretKeyName string, secretStore Cred, service string, pair UserPasswordPair) error {
	path := fmt.Sprintf("%s/%s/%s", secretBasePath, service, secretKeyName)
	existing, err := secretStore.AlreadyInStore(path)
	if err != nil {
		return err
	}
	if !existing {
		err = secretStore.UploadToStore(&pair, path)
		if err != nil {
			lc.Errorf("failed to upload credential pair for %s on path %s", service, path)
			return err
		}
	} else {
		lc.Infof("credentials for %s already present at path %s", service, path)
	}

	return err
}

func getCredential(credBootstrapStem string, cred Cred, service string) (UserPasswordPair, error) {
	path := fmt.Sprintf("%s/%s/%s", secretBasePath, credBootstrapStem, service)

	pair, err := cred.getUserPasswordPair(path)
	if err != nil {
		return UserPasswordPair{}, err
	}

	return *pair, err

}

func storeCredential(lc logger.LoggingClient, credBootstrapStem string, cred Cred, secretKeyName string, pair UserPasswordPair) error {
	path := fmt.Sprintf("%s/%s/%s", secretBasePath, credBootstrapStem, secretKeyName)
	existing, err := cred.AlreadyInStore(path)
	if err != nil {
		lc.Error(err.Error())
		return err
	}
	if !existing {
		err = cred.UploadToStore(&pair, path)
		if err != nil {
			lc.Errorf("failed to upload credential pair for %s on path %s", secretKeyName, path)
			return err
		}
	} else {
		lc.Infof("credentials for %s already present at path %s", secretKeyName, path)
	}

	return err
}

func LoadInitResponse(
	lc logger.LoggingClient,
	fileOpener fileioperformer.FileIoPerformer,
	secretConfig config.SecretStoreInfo,
	initResponse *types.InitResponse) error {

	absPath := filepath.Join(secretConfig.TokenFolderPath, secretConfig.TokenFile)

	tokenFile, err := fileOpener.OpenFileReader(absPath, os.O_RDONLY, 0400)
	if err != nil {
		lc.Errorf("could not read master key shares file %s: %s", absPath, err.Error())
		return err
	}
	tokenFileCloseable := fileioperformer.MakeReadCloser(tokenFile)
	defer func() { _ = tokenFileCloseable.Close() }()

	decoder := json.NewDecoder(tokenFileCloseable)
	if decoder == nil {
		err := errors.New("Failed to create JSON decoder")
		lc.Error(err.Error())
		return err
	}
	if err := decoder.Decode(initResponse); err != nil {
		lc.Errorf("unable to read token file at %s with error: %s", absPath, err.Error())
		return err
	}

	return nil
}

func saveInitResponse(
	lc logger.LoggingClient,
	fileOpener fileioperformer.FileIoPerformer,
	secretConfig config.SecretStoreInfo,
	initResponse *types.InitResponse) error {

	absPath := filepath.Join(secretConfig.TokenFolderPath, secretConfig.TokenFile)

	tokenFile, err := fileOpener.OpenFileWriter(absPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
	if err != nil {
		lc.Errorf("could not read master key shares file %s: %s", absPath, err.Error())
		return err
	}

	encoder := json.NewEncoder(tokenFile)
	if encoder == nil {
		err := errors.New("Failed to create JSON encoder")
		lc.Error(err.Error())
		_ = tokenFile.Close()
		return err
	}
	if err := encoder.Encode(initResponse); err != nil {
		lc.Errorf("unable to write token file at %s with error: %s", absPath, err.Error())
		_ = tokenFile.Close()
		return err
	}

	if err := tokenFile.Close(); err != nil {
		lc.Errorf("unable to close token file at %s with error: %s", absPath, err.Error())
		_ = tokenFile.Close()
		return err
	}

	return nil
}

// genPostgresCredentials generates the Postgres passwords for security-bootstrapper-postgres and other services
// and stores the passwords to secret store
func genPostgresCredentials(dic *di.Container, secretStore Cred, knownSecretsToAdd map[string][]string, ctx context.Context) error {
	configuration := container.ConfigurationFrom(dic.Get)
	lc := bootstrapContainer.LoggingClientFrom(dic.Get)

	// Add any additional services that need the known DB secret
	lc.Infof("adding any additional services using Postgres for knownSecrets...")
	services, ok := knownSecretsToAdd[postgresSecretName]
	if ok {
		for _, service := range services {
			password, err := secretStore.GeneratePassword(ctx)
			if err != nil {
				lc.Errorf("failed to generate Postgres password for service '%s'", service)
				return err
			}

			postgresCred := UserPasswordPair{
				User:     service,
				Password: password,
			}
			// store the Postgres credential to additional service (path /v1/secret/edgex/<additional-service-name>)
			err = addServiceCredential(lc, postgresSecretName, secretStore, service, postgresCred)
			if err != nil {
				lc.Error(err.Error())
				return err
			}

			// store the postgres credential to security-bootstrapper-postgres
			// (path /v1/secret/edgex/security-bootstrapper-postgres/<service-name>) as well
			err = storeCredential(lc, path.Join(common.SecurityBootstrapperPostgresKey, service), secretStore, postgresSecretName, postgresCred)
			if err != nil {
				lc.Error(err.Error())
				return err
			}
		}
	}

	lc.Infof("adding postgres secret name for internal services...")
	for key, info := range configuration.Databases {
		service := info.Service

		// add credentials to service path if specified and they're not already there
		if len(service) != 0 {
			password, err := secretStore.GeneratePassword(ctx)
			if err != nil {
				lc.Errorf("failed to generate Postgres password for service '%s'", key)
				return err
			}

			postgresCred := UserPasswordPair{
				User:     info.Username,
				Password: password,
			}
			// store the Postgres credential to EdgeX service (path /v1/secret/edgex/<service-name>)
			err = addServiceCredential(lc, postgresSecretName, secretStore, service, postgresCred)
			if err != nil {
				lc.Error(err.Error())
				return err
			}

			// store the postgres credential to security-bootstrapper-postgres
			// (path /v1/secret/edgex/security-bootstrapper-postgres/<service-name>) as well
			err = storeCredential(lc, path.Join(common.SecurityBootstrapperPostgresKey, service), secretStore, postgresSecretName, postgresCred)
			if err != nil {
				lc.Error(err.Error())
				return err
			}
		}
	}
	return nil
}