feat: Support splitting fogg config into directory with partials (#218)

* feat: Support splitting fogg config into directory with partials

- Add new `conf_dir` top level field
- Validate config directory and read all partial configs
- Merge partial configs into root config
- Add sample golden test case

* chore: Run make update-golden-files

Update test golden test files for split config

* fix: Failing CI tests

Update golden tests with hardcoded `fogg.d` config dir

apply/golden_file_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/chanzuckerberg/fogg/apply"
@@ -29,6 +30,7 @@ func TestIntegration(t *testing.T) {
 		{"snowflake_provider_yaml"},
 		{"v2_full_yaml"},
 		{"v2_minimal_valid_yaml"},
+		{"v2_split_yaml"},
 		{"v2_no_aws_provider_yaml"},
 		{"github_actions"},
 		{"github_actions_with_iam_role"},
@@ -54,9 +56,9 @@ func TestIntegration(t *testing.T) {
 			testdataFs := afero.NewBasePathFs(afero.NewOsFs(), filepath.Join(util.ProjectRoot(), "testdata", tt.fileName))
 			configFile := "fogg.yml"
 			if *updateGoldenFiles {
-				// delete all files except fogg.yml
+				// delete all files except fogg.yml and conf.d directory
 				e := afero.Walk(testdataFs, ".", func(path string, info os.FileInfo, err error) error {
-					if !info.IsDir() && !(path == configFile) {
+					if !info.IsDir() && !(path == configFile) && !(strings.Contains(path, "fogg.d")) {
 						return testdataFs.Remove(path)
 					}
 					return nil
@@ -84,6 +86,22 @@ func TestIntegration(t *testing.T) {
 				configMode, e := testdataFs.Stat(configFile)
 				r.NoError(e)
 				r.NoError(afero.WriteFile(fs, configFile, configContents, configMode.Mode()))
+				// if fogg.d exists, copy all partial configs too
+				confDir, e := testdataFs.Stat("fogg.d")
+				fs.Mkdir("fogg.d", 0700)
+				if e == nil && confDir.IsDir() {
+					afero.Walk(testdataFs, "fogg.d", func(path string, info os.FileInfo, err error) error {
+						if !info.IsDir() {
+							partialConfigContents, e := afero.ReadFile(testdataFs, path)
+							r.NoError(e)
+							partialConfigMode, e := testdataFs.Stat(configFile)
+							r.NoError(e)
+							r.NoError(afero.WriteFile(fs, path, partialConfigContents, partialConfigMode.Mode()))
+							return nil
+						}
+						return nil
+					})
+				}
 
 				conf, e := config.FindAndReadConfig(fs, configFile)
 				r.NoError(e)

config/v2/config.go

@@ -4,12 +4,16 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
+	"maps"
+	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/chanzuckerberg/fogg/errs"
 	"github.com/chanzuckerberg/fogg/plugins"
 	"github.com/runatlantis/atlantis/server/core/config/raw"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -32,12 +36,97 @@ func ReadConfig(fs afero.Fs, b []byte, configFile string) (*Config, error) {
 		decoder := yaml.NewDecoder(reader)
 		decoder.KnownFields(true)
 		e = decoder.Decode(c)
+		if e == nil && c.ConfDir != nil && *c.ConfDir != "" {
+			logrus.Debugf("Conf dir is %q\n", *c.ConfDir)
+			e = ReadConfDir(fs, c)
+		}
+
 	default:
 		return nil, errs.NewUserf("File type %s is not supported", ext)
 	}
 	return c, e
 }
 
+func ReadConfDir(fs afero.Fs, c *Config) error {
+	info, e := fs.Stat(*c.ConfDir)
+	if e != nil {
+		return errs.WrapUserf(e, "unable to find conf_dir %q", *c.ConfDir)
+	}
+	if !info.IsDir() {
+		return errs.WrapUserf(e, "conf_dir %q must be a directory", *c.ConfDir)
+	}
+	logrus.Debugf("Walking Conf dir %q\n", *c.ConfDir)
+	partialConfigs := []*Config{c}
+	e = afero.Walk(fs, *c.ConfDir, func(path string, info os.FileInfo, err error) error {
+		// TODO: ignore more files?
+		if info.IsDir() {
+			logrus.Debugf("Ignoring %q\n", path)
+			return nil
+		}
+		logrus.Debugf("Opening %q\n", path)
+		partial, e := fs.Open(path)
+		if e != nil {
+			logrus.Debugf("Ignoring error opening %q\n", path)
+			return nil
+		}
+		b, e := io.ReadAll(partial)
+		if e != nil {
+			return errs.WrapUserf(e, "unable to read partial config %q", path)
+		}
+		pc, e := ReadConfig(fs, b, path)
+		if e != nil {
+			return errs.WrapUserf(e, "unable to parse partial config %q", path)
+		}
+		logrus.Debugf("appending partialConfig %q\n", path)
+		partialConfigs = append(partialConfigs, pc)
+		return nil
+	})
+	if e != nil {
+		return errs.WrapUserf(e, "unable to walk conf_dir %q", *c.ConfDir)
+	}
+	// merge partialConfigs into c
+	mergeConfigs(partialConfigs...)
+	return e
+}
+
+func mergeConfigs(confs ...*Config) {
+	if len(confs) < 2 {
+		return
+	}
+	mergedConfig, tail := confs[0], confs[1:]
+	for _, pc := range tail {
+		if mergedConfig.Accounts == nil {
+			if pc.Accounts != nil {
+				mergedConfig.Accounts = pc.Accounts
+			}
+		} else {
+			if pc.Accounts != nil {
+				maps.Copy(mergedConfig.Accounts, pc.Accounts)
+			}
+		}
+
+		if mergedConfig.Envs == nil {
+			if pc.Envs != nil {
+				mergedConfig.Envs = pc.Envs
+			}
+		} else {
+			if pc.Envs != nil {
+				maps.Copy(mergedConfig.Envs, pc.Envs)
+			}
+		}
+
+		if mergedConfig.Modules == nil {
+			if pc.Modules != nil {
+				mergedConfig.Modules = pc.Modules
+			}
+		} else {
+			if pc.Modules != nil {
+				maps.Copy(mergedConfig.Modules, pc.Modules)
+			}
+		}
+	}
+}
+
 func (c *Config) Write(fs afero.Fs, path string) error {
 	yamlConfigFile, err := fs.Create("fogg.yml")
 	if err != nil {
@@ -60,6 +149,7 @@ type Config struct {
 	Plugins  Plugins            `yaml:"plugins,omitempty"`
 	Version  int                `validate:"required,eq=2"`
 	TFE      *TFE               `yaml:"tfe,omitempty"`
+	ConfDir  *string            `yaml:"conf_dir,omitempty"`
 }
 
 type TFE struct {

testdata/v2_split_yaml/.fogg-version

@@ -0,0 +1 @@
+undefined-pre+undefined.dirty

testdata/v2_split_yaml/.gitattributes

@@ -0,0 +1,11 @@
+fogg.tf linguist-generated
+remote-states.tf linguist-generated
+Makefile linguist-generated
+atlantis.yaml linguist-generated
+.travis.yml linguist-generated
+.circleci/config.yml linguist-generated
+.terraformignore linguist-generated
+.github/workflows/fogg_ci.yml linguist-generated
+.github/workflows/actions/setup-pre-commit/action.yml linguist-generated
+.pre-commit-config
+requirements.txt

testdata/v2_split_yaml/.gitignore

@@ -0,0 +1,38 @@
+# Auto-generated by fogg. Do not edit
+# Make improvements in fogg, so that everyone can benefit.
+
+# Compiled files
+*.tfstate
+*.tfstate.*.backup
+*.tfstate.backup
+*tfvars
+.terraform.lock.hcl
+
+# Module directory
+.terraform/
+
+# Pycharm folder
+.idea
+
+# Editor Swap Files
+*.swp
+*.swo
+*.swn
+*.swm
+*.swl
+*.swk
+
+.fogg
+/terraform.d/plugins
+/terraform.d/modules
+
+.DS_Store
+.vscode
+
+# Scala language server
+.metals
+
+buildevents.plan
+check-plan.output
+
+venv

testdata/v2_split_yaml/.terraform.d/plugin-cache/.gitignore

@@ -0,0 +1,5 @@
+# Auto-generated by fogg. Do not edit
+# Make improvements in fogg, so that everyone can benefit.
+
+*
+!.gitignore