Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[feature] Add a command to migrate from v1 config to v2 configs #270

Merged
merged 14 commits into from
May 30, 2019
40 changes: 40 additions & 0 deletions cmd/upgrade.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package cmd

import (
"os"

"github.com/chanzuckerberg/fogg/config"
"github.com/chanzuckerberg/fogg/errs"
"github.com/spf13/afero"
"github.com/spf13/cobra"
)

func init() {
upgrade.Flags().StringP("config", "c", "fogg.json", "Use this to override the fogg config file.")
rootCmd.AddCommand(upgrade)
}

var upgrade = &cobra.Command{
Use: "upgrade",
Short: "Upgrades a fogg config",
Long: `This command will upgrade a fogg config.
Note that this might be a lossy transformation.`,
RunE: func(cmd *cobra.Command, args []string) error {
// Set up fs
pwd, err := os.Getwd()
if err != nil {
return errs.WrapUser(err, "can't get pwd")
}
fs := afero.NewBasePathFs(afero.NewOsFs(), pwd)

// handle flags
configFile, err := cmd.Flags().GetString("config")
if err != nil {
return errs.WrapInternal(err, "couldn't parse config flag")
}

// check that we are at root of initialized git repo
openGitOrExit(fs)
return config.Upgrade(fs, configFile)
},
}
2 changes: 1 addition & 1 deletion cmd/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"strings"

"github.com/chanzuckerberg/fogg/config"
"github.com/chanzuckerberg/fogg/config/v2"
v2 "github.com/chanzuckerberg/fogg/config/v2"
"github.com/chanzuckerberg/fogg/errs"
"github.com/kr/pretty"
"github.com/sirupsen/logrus"
Expand Down
82 changes: 48 additions & 34 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import (
"io"
"io/ioutil"

"github.com/chanzuckerberg/fogg/config/v1"
"github.com/chanzuckerberg/fogg/config/v2"
v1 "github.com/chanzuckerberg/fogg/config/v1"
v2 "github.com/chanzuckerberg/fogg/config/v2"
"github.com/chanzuckerberg/fogg/errs"
"github.com/chanzuckerberg/fogg/util"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -35,25 +35,33 @@ func InitConfig(project, region, bucket, table, awsProfile, owner, awsProviderVe
}
}

func FindAndReadConfig(fs afero.Fs, configFile string) (*v2.Config, error) {
// FindConfig loads a config and its version into memory
func FindConfig(fs afero.Fs, configFile string) ([]byte, int, error) {
f, err := fs.Open(configFile)
if err != nil {
return nil, errs.WrapUser(err, "unable to open config file")
return nil, 0, errs.WrapUser(err, "unable to open config file")
}
reader := io.ReadCloser(f)
defer reader.Close()

b, e := ioutil.ReadAll(reader)
if e != nil {
return nil, errs.WrapUser(e, "unable to read config")
return nil, 0, errs.WrapUser(e, "unable to read config")
}

v, err := detectVersion(b)
if err != nil {
return nil, err
return nil, 0, err
}
logrus.Debugf("config file version: %#v\n", v)
return b, v, nil
}

func FindAndReadConfig(fs afero.Fs, configFile string) (*v2.Config, error) {
b, v, err := FindConfig(fs, configFile)
if err != nil {
return nil, err
}
switch v {
case 1:
c, err := v1.ReadConfig(b)
Expand All @@ -71,13 +79,13 @@ func FindAndReadConfig(fs afero.Fs, configFile string) (*v2.Config, error) {
default:
return nil, errs.NewUser("could not figure out config file version")
}

}

type ver struct {
Version int `json:"version"`
}

// detectVersion will detect the version of a config
func detectVersion(b []byte) (int, error) {
v := &ver{}
err := json.Unmarshal(b, v)
Expand Down Expand Up @@ -135,40 +143,37 @@ func UpgradeConfigVersion(c1 *v1.Config) (*v2.Config, error) {
}

for acctName, acct := range c1.Accounts {
c2.Accounts[acctName] = v2.Account{
Common: v2.Common{
Backend: &v2.Backend{
Bucket: acct.InfraBucket,
DynamoTable: acct.InfraDynamoTable,
Profile: acct.AWSProfileBackend,
Region: acct.AWSRegionBackend,
},
ExtraVars: acct.ExtraVars,
Providers: &v2.Providers{
AWS: &v2.AWSProvider{
AccountID: acct.AccountID,
AdditionalRegions: acct.AWSRegions,
Profile: acct.AWSProfileProvider,
Region: acct.AWSRegionProvider,
Version: acct.AWSProviderVersion,
},
common := v2.Common{
ExtraVars: acct.ExtraVars,
Providers: &v2.Providers{
AWS: &v2.AWSProvider{
AccountID: acct.AccountID,
AdditionalRegions: acct.AWSRegions,
Profile: acct.AWSProfileProvider,
Region: acct.AWSRegionProvider,
Version: acct.AWSProviderVersion,
},
Owner: acct.Owner,
Project: acct.Project,
TerraformVersion: acct.TerraformVersion,
},
Owner: acct.Owner,
Project: acct.Project,
TerraformVersion: acct.TerraformVersion,
}
if acct.InfraBucket != nil || acct.InfraDynamoTable != nil || acct.AWSProfileBackend != nil || acct.AWSRegionBackend != nil {
common.Backend = &v2.Backend{
Bucket: acct.InfraBucket,
DynamoTable: acct.InfraDynamoTable,
Profile: acct.AWSProfileBackend,
Region: acct.AWSRegionBackend,
}
}
c2.Accounts[acctName] = v2.Account{
Common: common,
}
}

for envName, env := range c1.Envs {
env2 := v2.Env{
Common: v2.Common{
Backend: &v2.Backend{
Bucket: env.InfraBucket,
DynamoTable: env.InfraDynamoTable,
Profile: env.AWSProfileBackend,
Region: env.AWSRegionBackend,
},
ExtraVars: env.ExtraVars,
Providers: &v2.Providers{
AWS: &v2.AWSProvider{
Expand All @@ -184,6 +189,16 @@ func UpgradeConfigVersion(c1 *v1.Config) (*v2.Config, error) {
TerraformVersion: env.TerraformVersion,
},
}

if env.InfraBucket != nil || env.InfraDynamoTable != nil || env.AWSProfileBackend != nil || env.AWSRegionBackend != nil {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we do this so we don't end up including a bunch of bucket: {} in the output

env2.Common.Backend = &v2.Backend{
Bucket: env.InfraBucket,
DynamoTable: env.InfraDynamoTable,
Profile: env.AWSProfileBackend,
Region: env.AWSRegionBackend,
}
}

env2.Components = map[string]v2.Component{}

for componentName, component := range env.Components {
Expand All @@ -209,7 +224,6 @@ func UpgradeConfigVersion(c1 *v1.Config) (*v2.Config, error) {
}

if component.AccountID != nil || component.AWSRegions != nil || component.AWSProfileProvider != nil || component.AWSRegionProvider != nil || component.TerraformVersion != nil {

c2.Providers = &v2.Providers{
AWS: &v2.AWSProvider{
AccountID: component.AccountID,
Expand Down
8 changes: 4 additions & 4 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"os"
"testing"

"github.com/chanzuckerberg/fogg/config/v1"
"github.com/chanzuckerberg/fogg/config/v2"
v1 "github.com/chanzuckerberg/fogg/config/v1"
v2 "github.com/chanzuckerberg/fogg/config/v2"
"github.com/chanzuckerberg/fogg/plugins"
"github.com/chanzuckerberg/fogg/util"
"github.com/go-test/deep"
Expand Down Expand Up @@ -227,7 +227,7 @@ func TestUpgradeConfigVersion(t *testing.T) {
"plugin": &plugins.CustomPlugin{
URL: "https://example.com/plugin.tgz",
Format: plugins.TypePluginFormatTar,
TarConfig: plugins.TarConfig{
TarConfig: &plugins.TarConfig{
StripComponents: 7,
},
},
Expand All @@ -236,7 +236,7 @@ func TestUpgradeConfigVersion(t *testing.T) {
"provider": &plugins.CustomPlugin{
URL: "https://example.com/provider.tgz",
Format: plugins.TypePluginFormatTar,
TarConfig: plugins.TarConfig{
TarConfig: &plugins.TarConfig{
StripComponents: 7,
},
},
Expand Down
42 changes: 42 additions & 0 deletions config/upgrade.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package config

import (
"encoding/json"

v1 "github.com/chanzuckerberg/fogg/config/v1"
"github.com/chanzuckerberg/fogg/errs"
"github.com/sirupsen/logrus"
"github.com/spf13/afero"
)

// Upgrade applies in-place upgrades to a configFile
func Upgrade(fs afero.Fs, configFile string) error {
bytes, version, err := FindConfig(fs, configFile)
if err != nil {
return err
}
switch version {
case 1:
c1, err := v1.ReadConfig(bytes)
if err != nil {
return err
}
c2, err := UpgradeConfigVersion(c1)
if err != nil {
return err
}

marshalled, err := json.MarshalIndent(c2, "", " ")
if err != nil {
return errs.WrapInternal(err, "Could not serialize config to json.")
}
err = afero.WriteFile(fs, configFile, marshalled, 0644)
return errs.WrapInternal(err, "Could not write config to disk")
case 2:
logrus.Infof("config already v%d, nothing to do", version)
return nil

default:
return errs.NewUserf("config version %d unrecognized", version)
}
}
59 changes: 59 additions & 0 deletions config/upgrade_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package config

import (
"testing"

"github.com/spf13/afero"

"github.com/stretchr/testify/require"

"github.com/chanzuckerberg/fogg/util"
)

func TestUpgradeV2(t *testing.T) {
r := require.New(t)
confPath := "config"
conf := []byte(`{"version": 1}`)
fs, _, err := util.TestFs()
r.Nil(err)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if it matters, but this might be better as NoError

err = afero.WriteFile(fs, confPath, conf, 0644)
r.Nil(err)
err = Upgrade(fs, confPath)
r.Nil(err)
}

func TestUpgradeUnknownVersion(t *testing.T) {
r := require.New(t)
confPath := "config"
conf := []byte(`{"version": 100}`)
fs, _, err := util.TestFs()
r.Nil(err)
err = afero.WriteFile(fs, confPath, conf, 0644)
r.Nil(err)
err = Upgrade(fs, confPath)
r.Error(err, "config version 100 unrecognized")
}

func TestUpgradeV1(t *testing.T) {
r := require.New(t)
confPath := "config"
fs, _, err := util.TestFs()
r.Nil(err)

v1, err := util.TestFile("v1_full")
r.NoError(err)

err = afero.WriteFile(fs, confPath, v1, 0644)
r.NoError(err)

_, v, err := FindConfig(fs, confPath)
r.NoError(err)
r.Equal(1, v)

err = Upgrade(fs, confPath)
r.NoError(err)

_, v, err = FindConfig(fs, confPath)
r.NoError(err)
r.Equal(2, v) // now upgraded
}
10 changes: 5 additions & 5 deletions config/v1/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,23 +122,23 @@ type Plugins struct {

// Module is a module
type Module struct {
TerraformVersion *string `json:"terraform_version"`
TerraformVersion *string `json:"terraform_version,omitempty"`
}

type TravisCI struct {
Enabled bool `json:"enabled"`
Enabled bool `json:"enabled,omitempty"`
AWSIAMRoleName string `json:"aws_iam_role_name"`
TestBuckets int `json:"test_buckets"`
}

type Config struct {
Accounts map[string]Account `json:"accounts"`
Defaults Defaults `json:"defaults"`
Docker bool `json:"docker"`
Docker bool `json:"docker,omitempty"`
Envs map[string]Env `json:"envs"`
Modules map[string]Module `json:"modules"`
Plugins Plugins `json:"plugins"`
TravisCI *TravisCI `json:"travis_ci"`
Plugins Plugins `json:"plugins,omitempty"`
TravisCI *TravisCI `json:"travis_ci,omitempty"`
}

func ReadConfig(b []byte) (*Config, error) {
Expand Down
10 changes: 5 additions & 5 deletions config/v2/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"math/rand"
"reflect"

"github.com/chanzuckerberg/fogg/config/v1"
v1 "github.com/chanzuckerberg/fogg/config/v1"
"github.com/chanzuckerberg/fogg/errs"
)

Expand Down Expand Up @@ -56,20 +56,20 @@ type Tools struct {
type Env struct {
Common

Components map[string]Component `json:"components"`
Components map[string]Component `json:"components,omitempty"`
}

type Component struct {
Common

EKS *v1.EKSConfig `json:"eks,omitempty"`
Kind *v1.ComponentKind `json:"kind,omitempty"`
ModuleSource *string `json:"module_source"`
ModuleSource *string `json:"module_source,omitempty"`
}

type Providers struct {
AWS *AWSProvider `json:"aws"`
Snowflake *SnowflakeProvider `json:"snowflake"`
AWS *AWSProvider `json:"aws,omitempty"`
Snowflake *SnowflakeProvider `json:"snowflake,omitempty"`
}

type AWSProvider struct {
Expand Down
Loading