diff --git a/cmd/upgrade.go b/cmd/upgrade.go new file mode 100644 index 000000000..b03d81f78 --- /dev/null +++ b/cmd/upgrade.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "os" + + "github.com/chanzuckerberg/fogg/config" + "github.com/chanzuckerberg/fogg/errs" + "github.com/spf13/afero" + "github.com/spf13/cobra" +) + +func init() { + upgrade.Flags().StringP("config", "c", "fogg.json", "Use this to override the fogg config file.") + rootCmd.AddCommand(upgrade) +} + +var upgrade = &cobra.Command{ + Use: "upgrade", + Short: "Upgrades a fogg config", + Long: `This command will upgrade a fogg config. + Note that this might be a lossy transformation.`, + RunE: func(cmd *cobra.Command, args []string) error { + // Set up fs + pwd, err := os.Getwd() + if err != nil { + return errs.WrapUser(err, "can't get pwd") + } + fs := afero.NewBasePathFs(afero.NewOsFs(), pwd) + + // handle flags + configFile, err := cmd.Flags().GetString("config") + if err != nil { + return errs.WrapInternal(err, "couldn't parse config flag") + } + + // check that we are at root of initialized git repo + openGitOrExit(fs) + return config.Upgrade(fs, configFile) + }, +} diff --git a/cmd/util.go b/cmd/util.go index 2419cf0f0..4bd7c7842 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/chanzuckerberg/fogg/config" - "github.com/chanzuckerberg/fogg/config/v2" + v2 "github.com/chanzuckerberg/fogg/config/v2" "github.com/chanzuckerberg/fogg/errs" "github.com/kr/pretty" "github.com/sirupsen/logrus" diff --git a/config/config.go b/config/config.go index 1d3f9c498..4ec049173 100644 --- a/config/config.go +++ b/config/config.go @@ -5,8 +5,8 @@ import ( "io" "io/ioutil" - "github.com/chanzuckerberg/fogg/config/v1" - "github.com/chanzuckerberg/fogg/config/v2" + v1 "github.com/chanzuckerberg/fogg/config/v1" + v2 "github.com/chanzuckerberg/fogg/config/v2" "github.com/chanzuckerberg/fogg/errs" "github.com/chanzuckerberg/fogg/util" "github.com/sirupsen/logrus" @@ -35,25 +35,33 @@ func InitConfig(project, region, bucket, table, awsProfile, owner, awsProviderVe } } -func FindAndReadConfig(fs afero.Fs, configFile string) (*v2.Config, error) { +// FindConfig loads a config and its version into memory +func FindConfig(fs afero.Fs, configFile string) ([]byte, int, error) { f, err := fs.Open(configFile) if err != nil { - return nil, errs.WrapUser(err, "unable to open config file") + return nil, 0, errs.WrapUser(err, "unable to open config file") } reader := io.ReadCloser(f) defer reader.Close() b, e := ioutil.ReadAll(reader) if e != nil { - return nil, errs.WrapUser(e, "unable to read config") + return nil, 0, errs.WrapUser(e, "unable to read config") } v, err := detectVersion(b) if err != nil { - return nil, err + return nil, 0, err } logrus.Debugf("config file version: %#v\n", v) + return b, v, nil +} +func FindAndReadConfig(fs afero.Fs, configFile string) (*v2.Config, error) { + b, v, err := FindConfig(fs, configFile) + if err != nil { + return nil, err + } switch v { case 1: c, err := v1.ReadConfig(b) @@ -71,13 +79,13 @@ func FindAndReadConfig(fs afero.Fs, configFile string) (*v2.Config, error) { default: return nil, errs.NewUser("could not figure out config file version") } - } type ver struct { Version int `json:"version"` } +// detectVersion will detect the version of a config func detectVersion(b []byte) (int, error) { v := &ver{} err := json.Unmarshal(b, v) @@ -135,40 +143,37 @@ func UpgradeConfigVersion(c1 *v1.Config) (*v2.Config, error) { } for acctName, acct := range c1.Accounts { - c2.Accounts[acctName] = v2.Account{ - Common: v2.Common{ - Backend: &v2.Backend{ - Bucket: acct.InfraBucket, - DynamoTable: acct.InfraDynamoTable, - Profile: acct.AWSProfileBackend, - Region: acct.AWSRegionBackend, - }, - ExtraVars: acct.ExtraVars, - Providers: &v2.Providers{ - AWS: &v2.AWSProvider{ - AccountID: acct.AccountID, - AdditionalRegions: acct.AWSRegions, - Profile: acct.AWSProfileProvider, - Region: acct.AWSRegionProvider, - Version: acct.AWSProviderVersion, - }, + common := v2.Common{ + ExtraVars: acct.ExtraVars, + Providers: &v2.Providers{ + AWS: &v2.AWSProvider{ + AccountID: acct.AccountID, + AdditionalRegions: acct.AWSRegions, + Profile: acct.AWSProfileProvider, + Region: acct.AWSRegionProvider, + Version: acct.AWSProviderVersion, }, - Owner: acct.Owner, - Project: acct.Project, - TerraformVersion: acct.TerraformVersion, }, + Owner: acct.Owner, + Project: acct.Project, + TerraformVersion: acct.TerraformVersion, + } + if acct.InfraBucket != nil || acct.InfraDynamoTable != nil || acct.AWSProfileBackend != nil || acct.AWSRegionBackend != nil { + common.Backend = &v2.Backend{ + Bucket: acct.InfraBucket, + DynamoTable: acct.InfraDynamoTable, + Profile: acct.AWSProfileBackend, + Region: acct.AWSRegionBackend, + } + } + c2.Accounts[acctName] = v2.Account{ + Common: common, } } for envName, env := range c1.Envs { env2 := v2.Env{ Common: v2.Common{ - Backend: &v2.Backend{ - Bucket: env.InfraBucket, - DynamoTable: env.InfraDynamoTable, - Profile: env.AWSProfileBackend, - Region: env.AWSRegionBackend, - }, ExtraVars: env.ExtraVars, Providers: &v2.Providers{ AWS: &v2.AWSProvider{ @@ -184,6 +189,16 @@ func UpgradeConfigVersion(c1 *v1.Config) (*v2.Config, error) { TerraformVersion: env.TerraformVersion, }, } + + if env.InfraBucket != nil || env.InfraDynamoTable != nil || env.AWSProfileBackend != nil || env.AWSRegionBackend != nil { + env2.Common.Backend = &v2.Backend{ + Bucket: env.InfraBucket, + DynamoTable: env.InfraDynamoTable, + Profile: env.AWSProfileBackend, + Region: env.AWSRegionBackend, + } + } + env2.Components = map[string]v2.Component{} for componentName, component := range env.Components { @@ -209,7 +224,6 @@ func UpgradeConfigVersion(c1 *v1.Config) (*v2.Config, error) { } if component.AccountID != nil || component.AWSRegions != nil || component.AWSProfileProvider != nil || component.AWSRegionProvider != nil || component.TerraformVersion != nil { - c2.Providers = &v2.Providers{ AWS: &v2.AWSProvider{ AccountID: component.AccountID, diff --git a/config/config_test.go b/config/config_test.go index 2ada7ea08..157a1e771 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -4,8 +4,8 @@ import ( "os" "testing" - "github.com/chanzuckerberg/fogg/config/v1" - "github.com/chanzuckerberg/fogg/config/v2" + v1 "github.com/chanzuckerberg/fogg/config/v1" + v2 "github.com/chanzuckerberg/fogg/config/v2" "github.com/chanzuckerberg/fogg/plugins" "github.com/chanzuckerberg/fogg/util" "github.com/go-test/deep" @@ -227,7 +227,7 @@ func TestUpgradeConfigVersion(t *testing.T) { "plugin": &plugins.CustomPlugin{ URL: "https://example.com/plugin.tgz", Format: plugins.TypePluginFormatTar, - TarConfig: plugins.TarConfig{ + TarConfig: &plugins.TarConfig{ StripComponents: 7, }, }, @@ -236,7 +236,7 @@ func TestUpgradeConfigVersion(t *testing.T) { "provider": &plugins.CustomPlugin{ URL: "https://example.com/provider.tgz", Format: plugins.TypePluginFormatTar, - TarConfig: plugins.TarConfig{ + TarConfig: &plugins.TarConfig{ StripComponents: 7, }, }, diff --git a/config/upgrade.go b/config/upgrade.go new file mode 100644 index 000000000..3531c6675 --- /dev/null +++ b/config/upgrade.go @@ -0,0 +1,42 @@ +package config + +import ( + "encoding/json" + + v1 "github.com/chanzuckerberg/fogg/config/v1" + "github.com/chanzuckerberg/fogg/errs" + "github.com/sirupsen/logrus" + "github.com/spf13/afero" +) + +// Upgrade applies in-place upgrades to a configFile +func Upgrade(fs afero.Fs, configFile string) error { + bytes, version, err := FindConfig(fs, configFile) + if err != nil { + return err + } + switch version { + case 1: + c1, err := v1.ReadConfig(bytes) + if err != nil { + return err + } + c2, err := UpgradeConfigVersion(c1) + if err != nil { + return err + } + + marshalled, err := json.MarshalIndent(c2, "", " ") + if err != nil { + return errs.WrapInternal(err, "Could not serialize config to json.") + } + err = afero.WriteFile(fs, configFile, marshalled, 0644) + return errs.WrapInternal(err, "Could not write config to disk") + case 2: + logrus.Infof("config already v%d, nothing to do", version) + return nil + + default: + return errs.NewUserf("config version %d unrecognized", version) + } +} diff --git a/config/upgrade_test.go b/config/upgrade_test.go new file mode 100644 index 000000000..4c25e246d --- /dev/null +++ b/config/upgrade_test.go @@ -0,0 +1,57 @@ +package config + +import ( + "testing" + + "github.com/chanzuckerberg/fogg/util" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestUpgradeV2(t *testing.T) { + r := require.New(t) + confPath := "config" + conf := []byte(`{"version": 1}`) + fs, _, err := util.TestFs() + r.Nil(err) + err = afero.WriteFile(fs, confPath, conf, 0644) + r.Nil(err) + err = Upgrade(fs, confPath) + r.Nil(err) +} + +func TestUpgradeUnknownVersion(t *testing.T) { + r := require.New(t) + confPath := "config" + conf := []byte(`{"version": 100}`) + fs, _, err := util.TestFs() + r.Nil(err) + err = afero.WriteFile(fs, confPath, conf, 0644) + r.Nil(err) + err = Upgrade(fs, confPath) + r.Error(err, "config version 100 unrecognized") +} + +func TestUpgradeV1(t *testing.T) { + r := require.New(t) + confPath := "config" + fs, _, err := util.TestFs() + r.Nil(err) + + v1, err := util.TestFile("v1_full") + r.NoError(err) + + err = afero.WriteFile(fs, confPath, v1, 0644) + r.NoError(err) + + _, v, err := FindConfig(fs, confPath) + r.NoError(err) + r.Equal(1, v) + + err = Upgrade(fs, confPath) + r.NoError(err) + + _, v, err = FindConfig(fs, confPath) + r.NoError(err) + r.Equal(2, v) // now upgraded +} diff --git a/config/v1/config.go b/config/v1/config.go index 59a8e41d2..bb5999908 100644 --- a/config/v1/config.go +++ b/config/v1/config.go @@ -122,11 +122,11 @@ type Plugins struct { // Module is a module type Module struct { - TerraformVersion *string `json:"terraform_version"` + TerraformVersion *string `json:"terraform_version,omitempty"` } type TravisCI struct { - Enabled bool `json:"enabled"` + Enabled bool `json:"enabled,omitempty"` AWSIAMRoleName string `json:"aws_iam_role_name"` TestBuckets int `json:"test_buckets"` } @@ -134,11 +134,11 @@ type TravisCI struct { type Config struct { Accounts map[string]Account `json:"accounts"` Defaults Defaults `json:"defaults"` - Docker bool `json:"docker"` + Docker bool `json:"docker,omitempty"` Envs map[string]Env `json:"envs"` Modules map[string]Module `json:"modules"` - Plugins Plugins `json:"plugins"` - TravisCI *TravisCI `json:"travis_ci"` + Plugins Plugins `json:"plugins,omitempty"` + TravisCI *TravisCI `json:"travis_ci,omitempty"` } func ReadConfig(b []byte) (*Config, error) { diff --git a/config/v2/config.go b/config/v2/config.go index 19a50b883..a8074a5e0 100644 --- a/config/v2/config.go +++ b/config/v2/config.go @@ -56,7 +56,7 @@ type Tools struct { type Env struct { Common - Components map[string]Component `json:"components"` + Components map[string]Component `json:"components,omitempty"` } type Component struct { @@ -64,13 +64,13 @@ type Component struct { EKS *v1.EKSConfig `json:"eks,omitempty"` Kind *v1.ComponentKind `json:"kind,omitempty"` - ModuleSource *string `json:"module_source"` + ModuleSource *string `json:"module_source,omitempty"` } type Providers struct { - AWS *AWSProvider `json:"aws"` - Snowflake *SnowflakeProvider `json:"snowflake"` - Bless *BlessProvider `json:"bless"` + AWS *AWSProvider `json:"aws,omitempty"` + Snowflake *SnowflakeProvider `json:"snowflake,omitempty"` + Bless *BlessProvider `json:"bless,omitempty"` } // BlessProvider allows for terraform-provider-bless configuration diff --git a/plugins/custom_plugin.go b/plugins/custom_plugin.go index 9c6187445..b9578d1d0 100644 --- a/plugins/custom_plugin.go +++ b/plugins/custom_plugin.go @@ -37,13 +37,20 @@ const ( type CustomPlugin struct { URL string `json:"url" validate:"required"` Format TypePluginFormat `json:"format" validate:"required"` - TarConfig TarConfig `json:"tar_config,omitempty"` - TargetDir string + TarConfig *TarConfig `json:"tar_config,omitempty"` + TargetDir string `json:"target_dir,omitempty"` } // TarConfig configures the tar unpacking type TarConfig struct { - StripComponents int `json:"strip_components"` + StripComponents int `json:"strip_components,omitempty"` +} + +func (tc *TarConfig) getStripComponents() int { + if tc == nil { + return 0 + } + return tc.StripComponents } // Install installs the custom plugin @@ -202,11 +209,11 @@ func (cp *CustomPlugin) processTar(fs afero.Fs, path string, targetDir string) e filepath.Clean(header.Name), string(os.PathSeparator)) // remove components if we can, otherwise skip this - if len(splitTarget) <= cp.TarConfig.StripComponents { + if len(splitTarget) <= cp.TarConfig.getStripComponents() { continue } target := filepath.Join(targetDir, - filepath.Join(splitTarget[cp.TarConfig.StripComponents:]...)) + filepath.Join(splitTarget[cp.TarConfig.getStripComponents():]...)) switch header.Typeflag { case tar.TypeDir: // if its a dir and it doesn't exist create it diff --git a/plugins/custom_plugin_test.go b/plugins/custom_plugin_test.go index 687c53273..15908a805 100644 --- a/plugins/custom_plugin_test.go +++ b/plugins/custom_plugin_test.go @@ -88,7 +88,7 @@ func TestCustomPluginTarStripComponents(t *testing.T) { customPlugin := &plugins.CustomPlugin{ URL: ts.URL, Format: plugins.TypePluginFormatTar, - TarConfig: plugins.TarConfig{ + TarConfig: &plugins.TarConfig{ StripComponents: 1, }, }