confmerger: log from internal structures instead of from files

Signed-off-by: Ridwan Sharif <[email protected]>

cmd/google_cloud_ops_agent_engine/main.go

@@ -16,13 +16,15 @@ package main
 
 import (
 	"flag"
-	"io/ioutil"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
 
 	"github.com/GoogleCloudPlatform/ops-agent/apps"
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
+	yaml "github.com/goccy/go-yaml"
+	"github.com/shirou/gopsutil/host"
 )
 
 var (
@@ -42,26 +44,30 @@ func main() {
 func run() error {
 	// TODO(lingshi) Move this to a shared place across Linux and Windows.
 	confDebugFolder := filepath.Join(os.Getenv("RUNTIME_DIRECTORY"), "conf", "debug")
-	if err := confgenerator.MergeConfFiles(*input, confDebugFolder, "linux", apps.BuiltInConfStructs); err != nil {
+	mergedConfig, err := confgenerator.MergeConfFiles(*input, confDebugFolder, "linux", apps.BuiltInConfStructs)
+	if err != nil {
 		return err
 	}
-	if err := logConfigFiles(filepath.Join(confDebugFolder, "built-in-config.yaml"), filepath.Join(confDebugFolder, "merged-config.yaml")); err != nil {
+
+	if err := logConfig(mergedConfig); err != nil {
 		return err
 	}
-	return confgenerator.GenerateFiles(filepath.Join(confDebugFolder, "merged-config.yaml"), *service, *logsDir, *stateDir, *outDir)
-}
 
-// logConfigFiles logs the built-in and merged config files to STDOUT. These are then written by journald to var/log/syslog and so to
-// Cloud Logging once the ops-agent is running.
-func logConfigFiles(builtInConfigFile, mergedConfigFile string) error {
-	builtInConfig, err := ioutil.ReadFile(builtInConfigFile)
+	hostInfo, _ := host.Info()
+	uc, err := confgenerator.ParseUnifiedConfigAndValidate(mergedConfig, hostInfo.OS)
 	if err != nil {
 		return err
 	}
+	return confgenerator.GenerateFilesFromConfig(&uc, *service, *logsDir, *stateDir, *outDir)
+}
 
-	mergedConfig, err := ioutil.ReadFile(mergedConfigFile)
+// logConfig logs the built-in and merged config files to STDOUT. These are then written by journald to var/log/syslog and so to
+// Cloud Logging once the ops-agent is running.
+func logConfig(mergedConfig []byte) error {
+	builtInStruct := apps.BuiltInConfStructs["linux"]
+	builtInConfig, err := yaml.Marshal(builtInStruct)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to convert the built-in config to yaml: %w \n", err)
 	}
 
 	log.Printf("Built-in config:\n%s", builtInConfig)

cmd/ops_agent_windows/run_windows.go

@@ -3,12 +3,12 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
 	"github.com/GoogleCloudPlatform/ops-agent/apps"
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
+	yaml "github.com/goccy/go-yaml"
 	"golang.org/x/sys/windows/svc"
 	"golang.org/x/sys/windows/svc/debug"
 	"golang.org/x/sys/windows/svc/eventlog"
@@ -113,17 +113,17 @@ func (s *service) checkForStandaloneAgents(unified *confgenerator.UnifiedConfig)
 func (s *service) generateConfigs() error {
 	// TODO(lingshi) Move this to a shared place across Linux and Windows.
 	confDebugFolder := filepath.Join(os.Getenv("PROGRAMDATA"), dataDirectory, "run", "conf", "debug")
-	if err := confgenerator.MergeConfFiles(s.userConf, confDebugFolder, "windows", apps.BuiltInConfStructs); err != nil {
-		return err
-	}
-	builtInConfig, err := ioutil.ReadFile(filepath.Join(confDebugFolder, "built-in-config.yaml"))
+	builtInStruct := apps.BuiltInConfStructs["windows"]
+	builtInConfig, err := yaml.Marshal(builtInStruct)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to convert the built-in config to yaml: %w \n", err)
 	}
-	mergedConfig, err := ioutil.ReadFile(filepath.Join(confDebugFolder, "merged-config.yaml"))
+
+	mergedConfig, err := confgenerator.MergeConfFiles(s.userConf, confDebugFolder, "windows", apps.BuiltInConfStructs)
 	if err != nil {
 		return err
 	}
+
 	s.log.Info(1, fmt.Sprintf("Built-in config:\n%s", builtInConfig))
 	s.log.Info(1, fmt.Sprintf("Merged config:\n%s", mergedConfig))
 	uc, err := confgenerator.ParseUnifiedConfigAndValidate(mergedConfig, "windows")

confgenerator/confgenerator_test.go

@@ -109,7 +109,7 @@ func testGenerateConfsWithValidInput(t *testing.T, platform platformConfig) {
 			userSpecifiedConfPath := filepath.Join(confDebugFolder, "/input.yaml")
 			builtInConfPath := filepath.Join(confDebugFolder, "/built-in-config.yaml")
 			mergedConfPath := filepath.Join(confDebugFolder, "/merged-config.yaml")
-			if err = confgenerator.MergeConfFiles(userSpecifiedConfPath, confDebugFolder, platform.OS, apps.BuiltInConfStructs); err != nil {
+			if _, err = confgenerator.MergeConfFiles(userSpecifiedConfPath, confDebugFolder, platform.OS, apps.BuiltInConfStructs); err != nil {
 				t.Fatalf("MergeConfFiles(%q, %q) got: %v", userSpecifiedConfPath, confDebugFolder, err)
 			}
 
@@ -227,7 +227,7 @@ func testGenerateConfigsWithInvalidInput(t *testing.T, platform platformConfig)
 			invalidInput := readFileContent(t, testName, platform.OS, invalidInputPath, false)
 			expectedError := readFileContent(t, testName, platform.OS, goldenErrorPath, true)
 
-			actualError := confgenerator.MergeConfFiles(userSpecifiedConfPath, confDebugFolder, platform.OS, apps.BuiltInConfStructs)
+			_, actualError := confgenerator.MergeConfFiles(userSpecifiedConfPath, confDebugFolder, platform.OS, apps.BuiltInConfStructs)
 			if actualError == nil {
 				mergedInput := readFileContent(t, testName, platform.OS, mergedInputPath, false)
 				actualError = generateConfigs(mergedInput, platform)

confgenerator/confmerger.go

@@ -17,62 +17,70 @@ package confgenerator
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
 	yaml "github.com/goccy/go-yaml"
 )
 
-func MergeConfFiles(userConfPath, confDebugFolder, platform string, builtInConfStructs map[string]*UnifiedConfig) error {
+// MergeConfFiles merges the user provided config with the built-in config struct for the platform.
+// It will write the merged config file to disk and return the config bytes.
+func MergeConfFiles(userConfPath, confDebugFolder, platform string, builtInConfStructs map[string]*UnifiedConfig) ([]byte, error) {
 	builtInConfPath := filepath.Join(confDebugFolder, "built-in-config.yaml")
 	mergedConfPath := filepath.Join(confDebugFolder, "merged-config.yaml")
-	return mergeConfFiles(builtInConfPath, userConfPath, mergedConfPath, platform, builtInConfStructs)
-}
 
-func mergeConfFiles(builtInConfPath, userConfPath, mergedConfPath, platform string, builtInConfStructs map[string]*UnifiedConfig) error {
+	// Write the built-in conf to disk for debugging purpose.
 	builtInStruct := builtInConfStructs[platform]
 	builtInYaml, err := yaml.Marshal(builtInStruct)
 	if err != nil {
-		return fmt.Errorf("failed to convert the built-in config %q to yaml: %w \n", builtInConfPath, err)
+		return nil, fmt.Errorf("failed to convert the built-in config %q to yaml: %w \n", builtInConfPath, err)
 	}
 
-	// Write the built-in conf to disk for debugging purpose.
-
 	if err := writeConfigFile(builtInYaml, builtInConfPath); err != nil {
-		return err
+		return nil, err
+	}
+
+	mergedConf, err := mergeConfFiles(builtInConfPath, userConfPath, mergedConfPath, platform, builtInConfStructs)
+	if err != nil {
+		return nil, err
+	}
+
+	mergedConfigYaml, err := yaml.Marshal(mergedConf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert the merged config %q to yaml: %w \n", mergedConfPath, err)
+	}
+	if err := writeConfigFile(mergedConfigYaml, mergedConfPath); err != nil {
+		return nil, err
 	}
 
+	return mergedConfigYaml, nil
+}
+
+func mergeConfFiles(builtInConfPath, userConfPath, mergedConfPath, platform string, builtInConfStructs map[string]*UnifiedConfig) (*UnifiedConfig, error) {
+	builtInStruct := builtInConfStructs[platform]
+
 	// Read the built-in config file.
 	original, err := builtInStruct.DeepCopy(platform)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	// Optionally merge the user config file.
 	if _, err = os.Stat(userConfPath); err != nil {
 		if os.IsNotExist(err) {
 			// Skip the merge if the user config file does not exist.
 		} else {
-			return fmt.Errorf("failed to retrieve the user config file %q: %w \n", userConfPath, err)
+			return nil, fmt.Errorf("failed to retrieve the user config file %q: %w \n", userConfPath, err)
 		}
 	} else {
 		overrides, err := ReadUnifiedConfigFromFile(userConfPath, platform)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		mergeConfigs(&original, &overrides)
 	}
 
-	// Write the merged conf file.
-	configBytes, err := yaml.Marshal(original)
-	if err != nil {
-		return fmt.Errorf("failed to convert the merged config %q to yaml: %w \n", mergedConfPath, err)
-	}
-	if err := ioutil.WriteFile(mergedConfPath, configBytes, 0644); err != nil {
-		return fmt.Errorf("failed to write the merged config file %q: %w \n", mergedConfPath, err)
-	}
-	return nil
+	return &original, nil
 }
 
 func mergeConfigs(original, overrides *UnifiedConfig) {