feat: Push common config to config provider

Signed-off-by: Elizabeth J Lee <[email protected]>

go.mod

@@ -3,10 +3,11 @@ module github.com/edgexfoundry/edgex-go
 require (
 	bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690
 	github.com/eclipse/paho.mqtt.golang v1.4.2
-	github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.12
+	github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.16
+	github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.2
 	github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0-dev.6
 	github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.4
-	github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.4
+	github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.5
 	github.com/fxamacker/cbor/v2 v2.4.0
 	github.com/golang-jwt/jwt/v4 v4.4.3
 	github.com/gomodule/redigo v1.8.9
@@ -27,7 +28,6 @@ require (
 	github.com/armon/go-metrics v0.3.10 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.2 // indirect
 	github.com/edgexfoundry/go-mod-registry/v3 v3.0.0-dev.3 // indirect
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect

go.sum

@@ -28,8 +28,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eclipse/paho.mqtt.golang v1.4.2 h1:66wOzfUHSSI1zamx7jR6yMEI5EuHnT1G6rNA5PM12m4=
 github.com/eclipse/paho.mqtt.golang v1.4.2/go.mod h1:JGt0RsEwEX+Xa/agj90YJ9d9DH2b7upDZMK9HRbFvCA=
-github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.12 h1:kLllnz4O5QRa2SPALhEIanzL20DtczsJYL5+NGKNz3o=
-github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.12/go.mod h1:JqdvZ2lSGydoHC++jlvfeawlB5ogFRCmWeYkoW+HHMo=
+github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.16 h1:MN6dOZHbYkW8JRDlEin/7T9nInOwpikZhthH0QaMksQ=
+github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.16/go.mod h1:G2C3aUWZ96nZU03XRvCBntU13eUjXGIB9KqRKrhI8h8=
 github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.2 h1:xp5MsP+qf/fuJxy8fT7k1N+c4j4C6w04qMCBXm6id7o=
 github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.2/go.mod h1:1Vv4uWAo6r7k6jUlqVJW8JOL6YKVBc6sRL8Al3DrMck=
 github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0-dev.6 h1:RQFs/HjVOi1X3YxJ8sm4vuX8nhKgH0caSf9RtjQvdeI=
@@ -38,8 +38,8 @@ github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.4 h1:swPZOjoQ/IUIWSJpZCmQ
 github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.4/go.mod h1:8pxuYvh2zcq1GuKqmk1MAuH1yuN40iOMmL0g2myIfwk=
 github.com/edgexfoundry/go-mod-registry/v3 v3.0.0-dev.3 h1:QgZF9f70Cwpvkjw3tP1aiVGHc+yNFJNzW6hO8pDs3fg=
 github.com/edgexfoundry/go-mod-registry/v3 v3.0.0-dev.3/go.mod h1:2w8v0sv+i21nY+DY6JV4PFxsNTuxpGAjlNFlFMTfZkk=
-github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.4 h1:IbgRyTFy7cDkXX4VZGeAnxKrTXkjQo3qrk+/UkYxug8=
-github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.4/go.mod h1:PU+4VEoB0MTETAM+m28bUXyU0dekzkkwusUuht6HQV4=
+github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.5 h1:tEo8BVH4OZuJ/q9ii1H4PdtxlXLh/kOKpRuWFTHOcBc=
+github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.5/go.mod h1:wYgPDiVaalEUzBkKoZox3OvcpY52GsMAGJWvX/H+MJg=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=

internal/core/common_config/main.go

@@ -16,21 +16,31 @@ package common_config
 
 import (
 	"context"
+	"fmt"
+	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/config"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/container"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/environment"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/flags"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/secret"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/startup"
 	"github.com/edgexfoundry/go-mod-bootstrap/v3/di"
+	"github.com/edgexfoundry/go-mod-configuration/v3/configuration"
+	"github.com/edgexfoundry/go-mod-configuration/v3/pkg/types"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/common"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/models"
+	"gopkg.in/yaml.v3"
 	"os"
 	"os/signal"
+	"sort"
 	"sync"
 	"syscall"
 )
 
+const (
+	commonConfigDone = "isCommonConfigReady"
+)
+
 func Main(ctx context.Context, cancel context.CancelFunc) {
 
 	// All common command-line flags have been moved to DefaultCommonFlags. Service specific flags can be add here,
@@ -56,23 +66,62 @@ func Main(ctx context.Context, cancel context.CancelFunc) {
 			return lc
 		},
 	})
-
 	secretProvider, err := secret.NewSecretProvider(nil, environment.NewVariables(lc), ctx, startupTimer, dic, common.CoreCommonConfigServiceKey)
 	if err != nil {
 		lc.Errorf("failed to create Secret Provider: %v", err)
 		os.Exit(1)
 	}
 
 	lc.Info("Secret Provider created")
+	var accessToken string
+	var getAccessToken types.GetAccessTokenCallback
+
+	if secretProvider != nil {
+		getAccessToken = func() (string, error) {
+			accessToken, err = secretProvider.GetAccessToken("consul", common.CoreCommonConfigServiceKey)
+			if err != nil {
+				return "", fmt.Errorf("failed to get Configuration Provider access token: %s", err.Error())
+			}
+			return accessToken, err
+		}
+		_, err = getAccessToken()
+		if err != nil {
+			lc.Errorf("failed to get Access Token for config provider: %s", err.Error())
+			os.Exit(1)
+		}
+	}
 
-	_, err = secretProvider.GetAccessToken("consul", common.CoreCommonConfigServiceKey)
+	lc.Info("Got Config Provider Access Token")
+	// create config client
+	envVars := environment.NewVariables(lc)
+	configProviderInfo, err := config.NewProviderInfo(envVars, f.ConfigProviderUrl())
+	if err != nil {
+		lc.Errorf("failed to get Provider Info for the common configuration: %s", err.Error())
+		os.Exit(1)
+	}
+	configClient, err := config.CreateProviderClient(lc, common.CoreCommonConfigServiceKey, common.ConfigStemCore, getAccessToken, configProviderInfo.ServiceConfig())
 	if err != nil {
-		lc.Errorf("failed to get Access Token for config provider: %v", err)
+		lc.Errorf("failed to create provider client for the common configuration: %s", err.Error())
 		os.Exit(1)
 	}
+	// check to see if the configuration exists in the config provider
+	hasConfig, err := configClient.HasConfiguration()
+	if err != nil {
+		lc.Errorf("failed to determine if configuration exists in the provider: %s", err.Error())
+		os.Exit(1)
+	}
+	lc.Infof("configuration exists: %v", hasConfig)
+	// load the yaml file and push it using the config client
+	if !hasConfig || f.OverwriteConfig() {
+		yamlFile := config.GetConfigLocation(lc, f)
+		lc.Infof("parsing %s for configuration", yamlFile)
+		err = loadYaml(lc, yamlFile, configClient)
+		if err != nil {
+			lc.Error(err.Error())
+			os.Exit(1)
+		}
+	}
 
-	lc.Info("Got Config Provider Access Token")
-	lc.Info("Core Common Config Ready for stage two")
 	lc.Info("Core Common Config exiting")
 	os.Exit(0)
 }
@@ -99,3 +148,74 @@ func translateInterruptToCancel(ctx context.Context, wg *sync.WaitGroup, cancel
 		}
 	}()
 }
+
+func loadYaml(lc logger.LoggingClient, yamlFile string, configClient configuration.Client) error {
+	// push not done flag to configClient
+	err := configClient.PutConfigurationValue(commonConfigDone, []byte("false"))
+	if err != nil {
+		return fmt.Errorf("failed to push %s on startup: %s", commonConfigDone, err.Error())
+	}
+	lc.Infof("reading %s", yamlFile)
+	contents, err := os.ReadFile(yamlFile)
+	if err != nil {
+		return fmt.Errorf("failed to read common configuration file %s: %s", yamlFile, err.Error())
+	}
+
+	data := make(map[string]interface{})
+	kv := make(map[string]interface{})
+
+	err = yaml.Unmarshal(contents, &data)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshall common configuration file %s: %s", yamlFile, err.Error())
+	}
+
+	kv = buildKeyValues(data, kv, "")
+
+	keys := make([]string, 0, len(kv))
+
+	for k := range kv {
+		keys = append(keys, k)
+	}
+
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		v := kv[k]
+		// Push key/value into Consul rather than print
+		err = configClient.PutConfigurationValue(k, []byte(fmt.Sprint(v)))
+		if err != nil {
+			return fmt.Errorf("failed to push common configuration key %s with value %v: %s", k, v, err.Error())
+		}
+	}
+
+	// push done flag to config client
+	err = configClient.PutConfigurationValue(commonConfigDone, []byte("true"))
+	if err != nil {
+		return fmt.Errorf("failed to push %s on completion: %s", commonConfigDone, err.Error())
+	}
+	return nil
+}
+
+// buildKeyValues is a helper function to parse the configuration yaml file
+func buildKeyValues(data map[string]interface{}, kv map[string]interface{}, origKey string) map[string]interface{} {
+	key := origKey
+	for k, v := range data {
+		if len(key) == 0 {
+			key = fmt.Sprintf("%s", k)
+		} else {
+			key = fmt.Sprintf("%s/%s", key, k)
+		}
+
+		vdata, ok := v.(map[string]interface{})
+		if !ok {
+			kv[key] = v
+			key = origKey
+			continue
+		}
+
+		kv = buildKeyValues(vdata, kv, key)
+		key = origKey
+	}
+
+	return kv
+}