Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

snowflake provider support #252

Merged
merged 10 commits into from
May 20, 2019
Merged

snowflake provider support #252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5d09a6c
add golden file test for v2
ryanking May 15, 2019
a2aaf99
move resolution logic to its own file
ryanking May 15, 2019
c74074b
remove unneded validations
ryanking May 15, 2019
fdeed25
refactor to allow configs with no aws provider
ryanking May 15, 2019
8d67933
[breaking] make aws provider optional
ryanking May 15, 2019
35a11eb
cleanup from self-review
ryanking May 15, 2019
d86a81e
more offline tests
ryanking May 16, 2019
7a40634
cleanup
ryanking May 16, 2019
edcdee9
add snowflake provider to fogg.tf templates
ryanking May 17, 2019
3658810
Merge remote-tracking branch 'origin/master' into ryanking/snowflake_…
ryanking May 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions apply/apply.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,34 +44,34 @@ func Apply(fs afero.Fs, conf *v2.Config, tmp *templates.T, upgrade bool) error {
return errs.WrapUser(err, "unable to evaluate plan")
}

e := applyRepo(fs, p, &tmp.Repo)
e := applyRepo(fs, p, &tmp.Repo, &tmp.Common)
if e != nil {
return errs.WrapUser(e, "unable to apply repo")
}

if p.TravisCI.Enabled {
e = applyTree(fs, &tmp.TravisCI, "", p.TravisCI)
e = applyTree(fs, &tmp.TravisCI, &tmp.Common, "", p.TravisCI)
if e != nil {
return errs.WrapUser(e, "unable to apply travis ci")
}
}

e = applyAccounts(fs, p, &tmp.Account)
e = applyAccounts(fs, p, &tmp.Account, &tmp.Common)
if e != nil {
return errs.WrapUser(e, "unable to apply accounts")
}

e = applyEnvs(fs, p, &tmp.Env, tmp.Components)
e = applyEnvs(fs, p, &tmp.Env, tmp.Components, &tmp.Common)
if e != nil {
return errs.WrapUser(e, "unable to apply envs")
}

e = applyGlobal(fs, p.Global, &tmp.Global)
e = applyGlobal(fs, p.Global, &tmp.Global, &tmp.Common)
if e != nil {
return errs.WrapUser(e, "unable to apply global")
}

e = applyModules(fs, p.Modules, &tmp.Module)
e = applyModules(fs, p.Modules, &tmp.Module, &tmp.Common)
return errs.WrapUser(e, "unable to apply modules")
}

Expand Down Expand Up @@ -107,51 +107,51 @@ func versionIsChanged(repo string, tool string) (bool, error) {
return toolVersion.NE(repoVersion), nil
}

func applyRepo(fs afero.Fs, p *plan.Plan, repoTemplates *packr.Box) error {
return applyTree(fs, repoTemplates, "", p)
func applyRepo(fs afero.Fs, p *plan.Plan, repoTemplates, commonTemplates *packr.Box) error {
return applyTree(fs, repoTemplates, commonTemplates, "", p)
}

func applyGlobal(fs afero.Fs, p plan.Component, repoBox *packr.Box) error {
func applyGlobal(fs afero.Fs, p plan.Component, repoBox, commonBox *packr.Box) error {
log.Debug("applying global")
path := fmt.Sprintf("%s/global", rootPath)
e := fs.MkdirAll(path, 0755)
if e != nil {
return errs.WrapUserf(e, "unable to make directory %s", path)
}
return applyTree(fs, repoBox, path, p)
return applyTree(fs, repoBox, commonBox, path, p)
}

func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox *packr.Box) (e error) {
func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox, commonBox *packr.Box) (e error) {
for account, accountPlan := range p.Accounts {
path := fmt.Sprintf("%s/accounts/%s", rootPath, account)
e = fs.MkdirAll(path, 0755)
if e != nil {
return errs.WrapUser(e, "unable to make directories for accounts")
}
e = applyTree(fs, accountBox, path, accountPlan)
e = applyTree(fs, accountBox, commonBox, path, accountPlan)
if e != nil {
return errs.WrapUser(e, "unable to apply templates to account")
}
}
return nil
}

func applyModules(fs afero.Fs, p map[string]plan.Module, moduleBox *packr.Box) (e error) {
func applyModules(fs afero.Fs, p map[string]plan.Module, moduleBox, commonBox *packr.Box) (e error) {
for module, modulePlan := range p {
path := fmt.Sprintf("%s/modules/%s", rootPath, module)
e = fs.MkdirAll(path, 0755)
if e != nil {
return errs.WrapUserf(e, "unable to make path %s", path)
}
e = applyTree(fs, moduleBox, path, modulePlan)
e = applyTree(fs, moduleBox, commonBox, path, modulePlan)
if e != nil {
return errs.WrapUser(e, "unable to apply tree")
}
}
return nil
}

func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBoxes map[v1.ComponentKind]packr.Box) (e error) {
func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBoxes map[v1.ComponentKind]packr.Box, commonBox *packr.Box) (e error) {
log.Debug("applying envs")
for env, envPlan := range p.Envs {
log.Debugf("applying %s", env)
Expand All @@ -160,7 +160,7 @@ func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBoxes map[
if e != nil {
return errs.WrapUserf(e, "unable to make directory %s", path)
}
e := applyTree(fs, envBox, path, envPlan)
e := applyTree(fs, envBox, commonBox, path, envPlan)
if e != nil {
return errs.WrapUser(e, "unable to apply templates to env")
}
Expand All @@ -171,13 +171,13 @@ func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBoxes map[
return errs.WrapUser(e, "unable to make directories for component")
}
componentBox := componentBoxes[componentPlan.Kind.GetOrDefault()]
e := applyTree(fs, &componentBox, path, componentPlan)
e := applyTree(fs, &componentBox, commonBox, path, componentPlan)
if e != nil {
return errs.WrapUser(e, "unable to apply templates for component")
}

if componentPlan.ModuleSource != nil {
e := applyModuleInvocation(fs, path, *componentPlan.ModuleSource, templates.Templates.ModuleInvocation)
e := applyModuleInvocation(fs, path, *componentPlan.ModuleSource, templates.Templates.ModuleInvocation, commonBox)
if e != nil {
return errs.WrapUser(e, "unable to apply module invocation")
}
Expand All @@ -188,15 +188,15 @@ func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBoxes map[
return nil
}

func applyTree(dest afero.Fs, source *packr.Box, targetBasePath string, subst interface{}) (e error) {
func applyTree(dest afero.Fs, source *packr.Box, common *packr.Box, targetBasePath string, subst interface{}) (e error) {
return source.Walk(func(path string, sourceFile packr.File) error {

extension := filepath.Ext(path)
target := getTargetPath(targetBasePath, path)
targetExtension := filepath.Ext(target)

if extension == ".tmpl" {
e = applyTemplate(sourceFile, dest, target, subst)
e = applyTemplate(sourceFile, common, dest, target, subst)
if e != nil {
return errs.WrapUser(e, "unable to apply template")
}
Expand Down Expand Up @@ -299,7 +299,7 @@ func removeExtension(path string) string {
return strings.TrimSuffix(path, filepath.Ext(path))
}

func applyTemplate(sourceFile io.Reader, dest afero.Fs, path string, overrides interface{}) error {
func applyTemplate(sourceFile io.Reader, commonTemplates *packr.Box, dest afero.Fs, path string, overrides interface{}) error {
dir, _ := filepath.Split(path)
ospath := filepath.FromSlash(dir)
err := dest.MkdirAll(ospath, 0775)
Expand All @@ -312,7 +312,7 @@ func applyTemplate(sourceFile io.Reader, dest afero.Fs, path string, overrides i
if err != nil {
return errs.WrapUser(err, "unable to open file")
}
t, e := util.OpenTemplate(sourceFile)
t, e := util.OpenTemplate(sourceFile, commonTemplates)
if e != nil {
return e
}
Expand All @@ -329,7 +329,7 @@ type moduleData struct {
Outputs []string
}

func applyModuleInvocation(fs afero.Fs, path, moduleAddress string, box packr.Box) error {
func applyModuleInvocation(fs afero.Fs, path, moduleAddress string, box packr.Box, commonBox *packr.Box) error {
e := fs.MkdirAll(path, 0755)
if e != nil {
return errs.WrapUserf(e, "couldn't create %s directory", path)
Expand Down Expand Up @@ -363,7 +363,7 @@ func applyModuleInvocation(fs afero.Fs, path, moduleAddress string, box packr.Bo
if e != nil {
return errs.WrapUser(e, "could not open template file")
}
e = applyTemplate(f, fs, filepath.Join(path, "main.tf"), &moduleData{moduleName, moduleAddressForSource, variables, outputs})
e = applyTemplate(f, commonBox, fs, filepath.Join(path, "main.tf"), &moduleData{moduleName, moduleAddressForSource, variables, outputs})
if e != nil {
return errs.WrapUser(e, "unable to apply template for main.tf")
}
Expand All @@ -378,7 +378,7 @@ func applyModuleInvocation(fs afero.Fs, path, moduleAddress string, box packr.Bo
return errs.WrapUser(e, "could not open template file")
}

e = applyTemplate(f, fs, filepath.Join(path, "outputs.tf"), &moduleData{moduleName, moduleAddressForSource, variables, outputs})
e = applyTemplate(f, commonBox, fs, filepath.Join(path, "outputs.tf"), &moduleData{moduleName, moduleAddressForSource, variables, outputs})
if e != nil {
return errs.WrapUser(e, "unable to apply template for outputs.tf")
}
Expand Down
8 changes: 4 additions & 4 deletions apply/apply_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func TestApplyTemplateBasic(t *testing.T) {
path := "bar"
overrides := struct{ Foo string }{"foo"}

e := applyTemplate(sourceFile, dest, path, overrides)
e := applyTemplate(sourceFile, &templates.Templates.Common, dest, path, overrides)
a.Nil(e)
f, e := dest.Open("bar")
a.Nil(e)
Expand All @@ -93,7 +93,7 @@ func TestApplyTemplateBasicNewDirectory(t *testing.T) {
path := filepath.Join(nonexistentDir, "bar")
overrides := struct{ Foo string }{"foo"}

e := applyTemplate(sourceFile, dest, path, overrides)
e := applyTemplate(sourceFile, &templates.Templates.Common, dest, path, overrides)
a.Nil(e)
f, e := dest.Open(path)
a.Nil(e)
Expand All @@ -112,7 +112,7 @@ func TestApplyTemplate(t *testing.T) {
path := "hello"
overrides := struct{ Name string }{"World"}

e := applyTemplate(sourceFile, dest, path, overrides)
e := applyTemplate(sourceFile, &templates.Templates.Common, dest, path, overrides)
a.Nil(e)
f, e := dest.Open("hello")
a.Nil(e)
Expand Down Expand Up @@ -288,7 +288,7 @@ func TestApplyModuleInvocation(t *testing.T) {

fs := afero.NewCopyOnWriteFs(pwdFs, testFs)

e := applyModuleInvocation(fs, "mymodule", "test-module", templates.Templates.ModuleInvocation)
e := applyModuleInvocation(fs, "mymodule", "test-module", templates.Templates.ModuleInvocation, &templates.Templates.Common)
a.NoError(e)

s, e := fs.Stat("mymodule")
Expand Down
1 change: 1 addition & 0 deletions apply/golden_file_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ func TestIntegration(t *testing.T) {
{"v2_full"},
{"v2_minimal_valid"},
{"v2_no_aws_provider"},
{"snowflake_provider"},
}

for _, tc := range testCases {
Expand Down
31 changes: 26 additions & 5 deletions config/v2/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ type Component struct {
}

type Providers struct {
AWS *AWSProvider `json:"aws"`
AWS *AWSProvider `json:"aws"`
Snowflake *SnowflakeProvider `json:"snowflake"`
}

type AWSProvider struct {
Expand All @@ -80,6 +81,12 @@ type AWSProvider struct {
Version *string `json:"version,omitempty"`
}

type SnowflakeProvider struct {
Account *string `json:"account,omitempty"`
Role *string `json:"role,omitempty"`
Region *string `json:"region,omitempty"`
}

type Backend struct {
Bucket *string `json:"bucket,omitempty"`
DynamoTable *string `json:"dynamodb_table,omitempty"`
Expand Down Expand Up @@ -146,16 +153,30 @@ func (c *Config) Generate(r *rand.Rand, size int) reflect.Value {
return nil
}

randSnowflakeProvider := func(r *rand.Rand, s int) *SnowflakeProvider {
if r.Float32() < 0.5 {
return &SnowflakeProvider{
Account: randStringPtr(r, size),
Region: randStringPtr(r, s),
Role: randStringPtr(r, s),
}
}
return nil
}

randCommon := func(r *rand.Rand, s int) Common {
c := Common{
Backend: &Backend{
Bucket: randStringPtr(r, s),
Region: randStringPtr(r, s),
},
ExtraVars: randStringMap(r, s),
Owner: randStringPtr(r, s),
Project: randStringPtr(r, s),
Providers: &Providers{AWS: randAWSProvider(r, s)},
ExtraVars: randStringMap(r, s),
Owner: randStringPtr(r, s),
Project: randStringPtr(r, s),
Providers: &Providers{
AWS: randAWSProvider(r, s),
Snowflake: randSnowflakeProvider(r, s),
},
TerraformVersion: randStringPtr(r, s),
}
return c
Expand Down
22 changes: 22 additions & 0 deletions config/v2/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/chanzuckerberg/fogg/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestReadConfig(t *testing.T) {
Expand All @@ -28,3 +29,24 @@ func TestReadConfig(t *testing.T) {
a.NoError(e)

}

func TestReadSnowflakeProvider(t *testing.T) {
r := require.New(t)

b, e := util.TestFile("snowflake_provider")
r.NoError(e)
r.NotNil(b)

c, e := ReadConfig(b)
r.NoError(e)
r.NotNil(c)

e = c.Validate()
r.NoError(e)

r.NotNil(c.Defaults.Providers)
r.NotNil(c.Defaults.Providers.Snowflake)
r.Equal("foo", *c.Defaults.Providers.Snowflake.Account)
r.Equal("bar", *c.Defaults.Providers.Snowflake.Role)
r.Equal("us-west-2", *c.Defaults.Providers.Snowflake.Region)
}
35 changes: 35 additions & 0 deletions config/v2/resolvers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ func ResolveStringMap(getter func(Common) map[string]string, commons ...Common)
// config objects passed in. Otherwise it will return nil.
func ResolveAWSProvider(commons ...Common) *AWSProvider {

// we may in the future want invert this implementation and walk the structs first
profile := lastNonNil(AWSProviderProfileGetter, commons...)
region := lastNonNil(AWSProviderRegionGetter, commons...)
version := lastNonNil(AWSProviderVersionGetter, commons...)
Expand All @@ -99,6 +100,21 @@ func ResolveAWSProvider(commons ...Common) *AWSProvider {
return nil
}

func ResolveSnowflakeProvider(commons ...Common) *SnowflakeProvider {
account := lastNonNil(SnowflakeProviderAccountGetter, commons...)
role := lastNonNil(SnowflakeProviderRoleGetter, commons...)
region := lastNonNil(SnowflakeProviderRegionGetter, commons...)

if account != nil || role != nil || region != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

below we make the assumption that if one is set then all are

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You mean below in this func? I don't think that's the case.

The current design thinking here is that the "resolution" logic will return the best version of the data we have, though it might be incomplete. Then validation will ensure that it is valid. Once we ensure that its valid we can confer the configs to plans knowing that all preconditions are met.

return &SnowflakeProvider{
Account: account,
Role: role,
Region: region,
}
}
return nil
}

func OwnerGetter(comm Common) *string {
return comm.Owner
}
Expand Down Expand Up @@ -186,3 +202,22 @@ func ResolveModuleTerraformVersion(def Defaults, module v1.Module) *string {
}
return def.TerraformVersion
}

func SnowflakeProviderAccountGetter(comm Common) *string {
if comm.Providers != nil && comm.Providers.Snowflake != nil {
return comm.Providers.Snowflake.Account
}
return nil
}
func SnowflakeProviderRoleGetter(comm Common) *string {
if comm.Providers != nil && comm.Providers.Snowflake != nil {
return comm.Providers.Snowflake.Role
}
return nil
}
func SnowflakeProviderRegionGetter(comm Common) *string {
if comm.Providers != nil && comm.Providers.Snowflake != nil {
return comm.Providers.Snowflake.Region
}
return nil
}
Loading