feat: Add env override capability for common configuration

cmd/core-common-config-bootstrapper/res/configuration.yaml

@@ -19,7 +19,7 @@ all-services:
         SecuritySecretsStored: false
         SecurityConsulTokensRequested: false
         SecurityConsulTokenDuration: false
-      Tags: # Contains the service level tags to be attached to all the service's metrics
+#     Tags: # Contains the service level tags to be attached to all the service's metrics
       #  Gateway: "my-iot-gateway" # Tag must be added here or via Consul Env Override can only change existing value, not added new ones.
 
   Service:

go.mod

@@ -2,12 +2,14 @@ module github.com/edgexfoundry/edgex-go
 
 go 1.20
 
+replace github.com/edgexfoundry/go-mod-bootstrap/v3 => ../MODS/go-mod-bootstrap
+
 require (
 	github.com/eclipse/paho.mqtt.golang v1.4.2
 	github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.46
 	github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.7
 	github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0-dev.27
-	github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.16
+	github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.18
 	github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.9
 	github.com/fxamacker/cbor/v2 v2.4.0
 	github.com/gomodule/redigo v1.8.9

go.sum

@@ -25,14 +25,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eclipse/paho.mqtt.golang v1.4.2 h1:66wOzfUHSSI1zamx7jR6yMEI5EuHnT1G6rNA5PM12m4=
 github.com/eclipse/paho.mqtt.golang v1.4.2/go.mod h1:JGt0RsEwEX+Xa/agj90YJ9d9DH2b7upDZMK9HRbFvCA=
-github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.46 h1:A15UzeUgFfQM+R27LvXeoUJAbVFgY41NEhMcDF8XOQU=
-github.com/edgexfoundry/go-mod-bootstrap/v3 v3.0.0-dev.46/go.mod h1:nweayvPW3G+nBZwwdRB6lWq3u0e1rWGvReEWEjDQjFE=
 github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.7 h1:pOH2GLDg1KB4EmAzo6IEvl4NEVFAw3ywxPUMa5Wi1Vw=
 github.com/edgexfoundry/go-mod-configuration/v3 v3.0.0-dev.7/go.mod h1:ZZbOu7K0/P8B1VKhZygVujLQyhvWuPe0E2vC/k2yscw=
 github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0-dev.27 h1:lKM/NwRBL/pddoSnlXbFIlpgu+XozWbqz608EeB7S94=
 github.com/edgexfoundry/go-mod-core-contracts/v3 v3.0.0-dev.27/go.mod h1:QBzXOAoyLzBm5k9CshgH9CR9nWMNXxZjknBFMnGJN/A=
-github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.16 h1:JdZiqZ6CAboZ3+GuWZLyXc42y8HfsH1CmsIKNPY+0PY=
-github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.16/go.mod h1:3q+Ys51HwpR9/kOv6sT7+EwAANgOfFWi509sEP5iPEo=
+github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.18 h1:8empqTOIVhKzebC3gnP9C0LGQvwXtHLJF8x8AqlTGLM=
+github.com/edgexfoundry/go-mod-messaging/v3 v3.0.0-dev.18/go.mod h1:Sf7lvKtICWRy0Tmvw+bV6L1wdXKRaTSmKmJKw4Caxe4=
 github.com/edgexfoundry/go-mod-registry/v3 v3.0.0-dev.5 h1:FUUfsQUKHJqSwOXEj0j/qAn+uoWzM8nV7IOapGf7D80=
 github.com/edgexfoundry/go-mod-registry/v3 v3.0.0-dev.5/go.mod h1:MroNeX/i6ymRu8cyuRB4U5zdBusyUpFhUphko6pInTQ=
 github.com/edgexfoundry/go-mod-secrets/v3 v3.0.0-dev.9 h1:12UHZG/Nl+dfFOaNnC87ANP/FQ3rOXe0Z6NVlYdmYgU=

internal/core/common_config/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/clients/logger"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/common"
 	"github.com/edgexfoundry/go-mod-core-contracts/v3/models"
+	"github.com/pelletier/go-toml"
 	"gopkg.in/yaml.v3"
 )
 
@@ -44,7 +45,7 @@ const (
 func Main(ctx context.Context, cancel context.CancelFunc) {
 	startupTimer := startup.NewStartUpTimer(common.CoreCommonConfigServiceKey)
 
-	// All common command-line flags have been moved to DefaultCommonFlags. Service specific flags can be add here,
+	// All common command-line flags have been moved to DefaultCommonFlags. Service specific flags can be added here,
 	// by inserting service specific flag prior to call to commonFlags.Parse().
 	// Example:
 	// 		flags.FlagSet.StringVar(&myvar, "m", "", "Specify a ....")
@@ -117,17 +118,18 @@ func Main(ctx context.Context, cancel context.CancelFunc) {
 		os.Exit(1)
 	}
 
-	lc.Infof("configuration exists: %v", hasConfig)
+	lc.Infof("Common configuration exists in Config Provider is %v and overwrite flag is %v", hasConfig, f.OverwriteConfig())
 
 	// load the yaml file and push it using the config client
 	if !hasConfig || f.OverwriteConfig() {
 		yamlFile := config.GetConfigLocation(lc, f)
-		lc.Infof("parsing %s for configuration", yamlFile)
-		err = loadYaml(lc, yamlFile, configClient)
+		err = pushConfiguration(lc, yamlFile, configClient)
 		if err != nil {
 			lc.Error(err.Error())
 			os.Exit(1)
 		}
+	} else {
+		lc.Infof("Skipped pushing common configuration")
 	}
 
 	lc.Info("Core Common Config exiting")
@@ -157,13 +159,13 @@ func translateInterruptToCancel(ctx context.Context, wg *sync.WaitGroup, cancel
 	}()
 }
 
-func loadYaml(lc logger.LoggingClient, yamlFile string, configClient configuration.Client) error {
+func pushConfiguration(lc logger.LoggingClient, yamlFile string, configClient configuration.Client) error {
 	// push not done flag to configClient
 	err := configClient.PutConfigurationValue(commonConfigDone, []byte("false"))
 	if err != nil {
 		return fmt.Errorf("failed to push %s on startup: %s", commonConfigDone, err.Error())
 	}
-	lc.Infof("reading %s", yamlFile)
+	lc.Infof("Using common configuration from %s", yamlFile)
 	contents, err := os.ReadFile(yamlFile)
 	if err != nil {
 		return fmt.Errorf("failed to read common configuration file %s: %s", yamlFile, err.Error())
@@ -179,6 +181,11 @@ func loadYaml(lc logger.LoggingClient, yamlFile string, configClient configurati
 
 	kv = buildKeyValues(data, kv, "")
 
+	kv, err = applyEnvOverrides(kv, lc)
+	if err != nil {
+		return fmt.Errorf("failed to apply env overrides to common configuration: %s", err.Error())
+	}
+
 	keys := make([]string, 0, len(kv))
 
 	for k := range kv {
@@ -203,6 +210,9 @@ func loadYaml(lc logger.LoggingClient, yamlFile string, configClient configurati
 	if err != nil {
 		return fmt.Errorf("failed to push %s on completion: %s", commonConfigDone, err.Error())
 	}
+
+	lc.Info("Common configuration has been pushed to into Configuration Provider with overrides applied")
+
 	return nil
 }
 
@@ -229,3 +239,21 @@ func buildKeyValues(data map[string]interface{}, kv map[string]interface{}, orig
 
 	return kv
 }
+
+func applyEnvOverrides(keyValues map[string]any, lc logger.LoggingClient) (map[string]any, error) {
+	env := environment.NewVariables(lc)
+	tomlTree, err := toml.TreeFromMap(keyValues)
+	if err != nil {
+		return nil, err
+	}
+
+	overrideCount, err := env.OverrideTomlValues(tomlTree)
+	if err != nil {
+		return nil, err
+	}
+
+	keyValues = tomlTree.ToMap()
+	lc.Infof("Common configuration loaded from file with %d overrides applied", overrideCount)
+
+	return keyValues, nil
+}