[feature] atlantis integration (chanzuckerberg#303)

[feature] atlantis integrationThis PR adds Atlantis (<runatlantis.io>) integration to fogg. The goal is to be able to use fogg to generate an atlantis.yaml configuration file and all related tools, scripts etc.

Happy to take feedback on both the approach and the implementation.

### Approach

The basic approach is, in no particular order–

* To the degree possible it would be great to be able to make changes to how Atlantis works without touching the Atlantis instances, so as much as possible is put into the per-repo config file.
* We allow configuring Atlantis on a per-component basis. I think this is important as I would expect teams to adopt it incrementally.
* We do a 2-phase plan - first . plan for each component (so that we can set a var in each) and then walk those plans and generate a plan for the atlantis.yaml and aws config
* We generate an aws config for atlantis. Because we rely on aws profiles for authentication, we need to find a way to get the Atlantis instance configured with all the right profiles. In the future we may want to think about doing this for other modes.
* If atlantis is enabled for a component, `make apply` will by default fail. You have to add `FORCE=1` to override. This should nudge people in the right direction while allowing a fall-back to running locally (important for adoption phase).
* we add a `make setup` that will install fogg and run `fogg setup` (you can run this from components, not just top-level)
* we no longer run `terraform get` since that is redundant with `terrraform init` and we **don't run -update=true anymore** which means that modules will not get updated. People should be using stable versions anyway and this makes it much faster
* tfenv is now installed inside the repo, rather than homedir. This makes it so that we can use tfenv in atlantis (rather than atlantis' own process for getting terraform versions)

--- 

Note there are also a handful of stylistic changes in this diff. Tried to keep those to a minimum, but also wanted to make some improvements, especially to our makefiles.

## Test Plan
* unit tests
* ran on shared-infra repo for 1 component

## References
* <https://www.runatlantis.io/docs/>
* Fixes chanzuckerberg#290 
* Fixes chanzuckerberg#289
* Fixes chanzuckerberg#278

Makefile

@@ -18,10 +18,14 @@ lint: ## run the fast go linters
 	golangci-lint run
 .PHONY: lint
 
-packr: ## run the packr tool to generate our static files
+TEMPLATES := $(shell find templates -not -name "*.go")
+
+templates/a_templates-packr.go: $(TEMPLATES)
 	packr clean -v
 	packr -v
 
+packr: templates/a_templates-packr.go ## run the packr tool to generate our static files
+
 release: ## run a release
 	./bin/bff bump
 	git push
@@ -64,6 +68,7 @@ help: ## display help for this makefile
 clean: ## clean the repo
 	rm fogg 2>/dev/null || true
 	go clean
+	go clean -testcache
 	rm -rf dist 2>/dev/null || true
 	packr clean
 	rm coverage.out 2>/dev/null || true

apply/apply.go

@@ -56,6 +56,13 @@ func Apply(fs afero.Fs, conf *v2.Config, tmp *templates.T, upgrade bool) error {
 		}
 	}
 
+	if p.Atlantis.Enabled {
+		e = applyTree(fs, &tmp.Atlantis, &tmp.Common, "", p.Atlantis)
+		if e != nil {
+			return errs.WrapUser(e, "unable to apply atlantis")
+		}
+	}
+
 	e = applyAccounts(fs, p, &tmp.Account, &tmp.Common)
 	if e != nil {
 		return errs.WrapUser(e, "unable to apply accounts")

apply/golden_file_test.go

@@ -112,7 +112,7 @@ func TestIntegration(t *testing.T) {
 						logrus.Debugf("f1:\n%s\n\n---- ", f1)
 						logrus.Debugf("f2:\n%s\n\n---- ", f2)
 
-						r.Equal(f1, f2)
+						r.Equal(f1, f2, path)
 					}
 					return nil
 				}))

config/v2/config.go

@@ -49,10 +49,17 @@ type Account struct {
 }
 
 type Tools struct {
+	Atlantis *Atlantis    `json:"atlantis,omitempty"`
 	TravisCI *v1.TravisCI `json:"travis_ci,omitempty"`
 	TfLint   *v1.TfLint   `json:"tflint,omitempty"`
 }
 
+type Atlantis struct {
+	Enabled  *bool   `json:"enabled,omitempty"`
+	RoleName *string `json:"role_name,omitempty"`
+	RolePath *string `json:"role_path,omitempty"`
+}
+
 type Env struct {
 	Common
 
@@ -107,6 +114,7 @@ type SnowflakeProvider struct {
 }
 
 type Backend struct {
+	AccountID   *string `json:"account_id,omitempty"`
 	Bucket      *string `json:"bucket,omitempty"`
 	DynamoTable *string `json:"dynamodb_table,omitempty"`
 	Profile     *string `json:"profile,omitempty"`
@@ -228,6 +236,14 @@ func (c *Config) Generate(r *rand.Rand, size int) reflect.Value {
 					Enabled: &p,
 				}
 			}
+			if r.Float32() < 0.5 {
+				t := true
+				c.Tools.Atlantis = &Atlantis{
+					Enabled:  &t,
+					RolePath: randStringPtr(r, s),
+					RoleName: randStringPtr(r, s),
+				}
+			}
 		}
 
 		return c

config/v2/config_test.go

@@ -96,3 +96,25 @@ func TestReadOktaProvider(t *testing.T) {
 	r.Equal("aversion", *c.Defaults.Providers.Okta.Version)
 	r.Equal("orgname", *c.Defaults.Providers.Okta.OrgName)
 }
+
+func TestReadAtlantis(t *testing.T) {
+	r := require.New(t)
+
+	b, e := util.TestFile("v2_full")
+	r.NoError(e)
+	r.NotNil(b)
+
+	c, e := ReadConfig(b)
+	r.NoError(e)
+	r.NotNil(c)
+
+	w, e := c.Validate()
+	r.NoError(e)
+	r.Len(w, 0)
+
+	r.NotNil(c.Defaults.Tools)
+	r.NotNil(c.Defaults.Tools.Atlantis)
+	r.True(*c.Defaults.Tools.Atlantis.Enabled)
+	r.Equal("foo", *c.Defaults.Tools.Atlantis.RolePath)
+	r.Equal("bar", *c.Defaults.Tools.Atlantis.RoleName)
+}

config/v2/resolvers.go

@@ -144,6 +144,7 @@ func ResolveOktaProvider(commons ...Common) *OktaProvider {
 		Version: lastNonNil(OktaProviderVersionGetter, commons...),
 	}
 }
+
 func ResolveBlessProvider(commons ...Common) *BlessProvider {
 	profile := lastNonNil(BlessProviderProfileGetter, commons...)
 	region := lastNonNil(BlessProviderRegionGetter, commons...)
@@ -175,6 +176,24 @@ func ResolveTfLint(commons ...Common) v1.TfLint {
 	}
 }
 
+func ResolveAtlantis(commons ...Common) *Atlantis {
+	enabled := false
+	for _, c := range commons {
+		if c.Tools != nil && c.Tools.Atlantis != nil && c.Tools.Atlantis.Enabled != nil {
+			enabled = *c.Tools.Atlantis.Enabled
+		}
+	}
+
+	roleName := lastNonNil(AtlantisRoleNameGetter, commons...)
+	rolePath := lastNonNil(AtlantisRolePathGetter, commons...)
+
+	return &Atlantis{
+		Enabled:  &enabled,
+		RoleName: roleName,
+		RolePath: rolePath,
+	}
+}
+
 func OwnerGetter(comm Common) *string {
 	return comm.Owner
 }
@@ -193,6 +212,14 @@ func BackendBucketGetter(comm Common) *string {
 	}
 	return nil
 }
+
+func BackendAccountIdGetter(comm Common) *string {
+	if comm.Backend != nil {
+		return comm.Backend.AccountID
+	}
+	return nil
+}
+
 func BackendRegionGetter(comm Common) *string {
 	if comm.Backend != nil {
 		return comm.Backend.Region
@@ -326,3 +353,17 @@ func OktaProviderOrgNameGetter(comm Common) *string {
 	}
 	return comm.Providers.Okta.OrgName
 }
+
+func AtlantisRolePathGetter(comm Common) *string {
+	if comm.Tools == nil || comm.Tools.Atlantis == nil {
+		return nil
+	}
+	return comm.Tools.Atlantis.RolePath
+}
+
+func AtlantisRoleNameGetter(comm Common) *string {
+	if comm.Tools == nil || comm.Tools.Atlantis == nil {
+		return nil
+	}
+	return comm.Tools.Atlantis.RoleName
+}

config/v2/validation.go

@@ -48,6 +48,7 @@ func (c *Config) Validate() ([]string, error) {
 	errs = multierror.Append(errs, c.ValidateBlessProviders())
 	errs = multierror.Append(errs, c.ValidateOktaProviders())
 	errs = multierror.Append(errs, c.validateModules())
+	errs = multierror.Append(errs, c.ValidateAtlantis())
 
 	// refactor to make it easier to manage these
 	w, e := c.ValidateToolsTravis()
@@ -64,10 +65,6 @@ func (c *Config) Validate() ([]string, error) {
 	return warnings, errs.ErrorOrNil()
 }
 
-func merge(warnings []string, err *multierror.Error, w []string, e error) ([]string, *multierror.Error) {
-	return append(warnings, w...), multierror.Append(err, e)
-}
-
 func ValidateAWSProvider(p *AWSProvider, component string) error {
 	var errs *multierror.Error
 	if p == nil {
@@ -173,6 +170,25 @@ func (c *Config) ValidateOktaProviders() error {
 	return errs
 }
 
+func (c *Config) ValidateAtlantis() error {
+	var errs *multierror.Error
+	c.WalkComponents(func(component string, comms ...Common) {
+		a := ResolveAtlantis(comms...)
+
+		if a.Enabled != nil && *a.Enabled {
+			if a.RoleName == nil || *a.RoleName == "" {
+				errs = multierror.Append(errs, fmt.Errorf("if atlantis is enabled, role name must be specified (currently %#v in %s)", a.RoleName, component))
+			}
+
+			backendAccountId := lastNonNil(BackendAccountIdGetter, comms...)
+			if backendAccountId == nil {
+				errs = multierror.Append(errs, fmt.Errorf("if atlantis is enabled, backend acccount id must be specified (%s)", component))
+			}
+		}
+	})
+	return errs
+}
+
 func (c *Config) ValidateBlessProviders() error {
 	var errs *multierror.Error
 	c.WalkComponents(func(component string, comms ...Common) {

plan/atlantis.go

@@ -0,0 +1,132 @@
+package plan
+
+import (
+	"fmt"
+	"sort"
+)
+
+type Atlantis struct {
+	Enabled  bool
+	Projects []AtlantisProject
+
+	// AWSProfiles is a map of profile name -> role info
+	// TODO de-dupe this with AWSProfile in the travis-ci plan
+	AWSProfiles map[string]AWSRole
+}
+
+type AtlantisProject struct {
+	Name             string `yaml:"name"`
+	Dir              string `yaml:"dir"`
+	TerraformVersion string `yaml:"terraform_version"`
+	PathToRepoRoot   string `yaml:"path_to_repo_root"`
+}
+
+type AWSRole struct {
+	AccountID string `yaml:"account_id"`
+	RolePath  string `yaml:"role_path"`
+	RoleName  string `yaml:"role_name"`
+}
+
+// buildAtlantis will walk all the components and build an atlantis plan
+func (p *Plan) buildAtlantis() Atlantis {
+	// TODO This func has a lot of duplication.
+	enabled := false
+	projects := []AtlantisProject{}
+	profiles := map[string]AWSRole{}
+
+	if p.Global.Atlantis.Enabled {
+		enabled = true
+		proj := AtlantisProject{
+			Name:             "global",
+			Dir:              "terraform/global",
+			PathToRepoRoot:   p.Global.PathToRepoRoot,
+			TerraformVersion: p.Global.TerraformVersion,
+		}
+
+		projects = append(projects, proj)
+
+		profiles[p.Global.Backend.Profile] = AWSRole{
+			AccountID: *p.Global.Backend.AccountID,
+			RoleName:  p.Global.Atlantis.RoleName,
+			RolePath:  p.Global.Atlantis.RolePath,
+		}
+
+		if p.Global.Providers.AWS != nil {
+			a := *p.Global.Providers.AWS
+			profiles[a.Profile] = AWSRole{
+				AccountID: *p.Global.Backend.AccountID,
+				RoleName:  p.Global.Atlantis.RoleName,
+				RolePath:  p.Global.Atlantis.RolePath,
+			}
+		}
+	}
+
+	for name, acct := range p.Accounts {
+		if acct.Atlantis.Enabled {
+			enabled = true
+			proj := AtlantisProject{
+				Name:             fmt.Sprintf("accounts/%s", name),
+				Dir:              fmt.Sprintf("terraform/accounts/%s", name),
+				PathToRepoRoot:   acct.PathToRepoRoot,
+				TerraformVersion: acct.TerraformVersion,
+			}
+			projects = append(projects, proj)
+
+			profiles[acct.Backend.Profile] = AWSRole{
+				AccountID: *acct.Backend.AccountID,
+				RoleName:  acct.Atlantis.RoleName,
+				RolePath:  acct.Atlantis.RolePath,
+			}
+
+			if acct.Providers.AWS != nil {
+				a := *acct.Providers.AWS
+				profiles[a.Profile] = AWSRole{
+					AccountID: a.AccountID.String(),
+					RoleName:  acct.Atlantis.RoleName,
+					RolePath:  acct.Atlantis.RolePath,
+				}
+			}
+		}
+	}
+
+	for envName, env := range p.Envs {
+		for cName, c := range env.Components {
+			if c.Atlantis.Enabled {
+				enabled = true
+				p := AtlantisProject{
+					Name:             fmt.Sprintf("%s/%s", envName, cName),
+					Dir:              fmt.Sprintf("terraform/envs/%s/%s", envName, cName),
+					PathToRepoRoot:   c.PathToRepoRoot,
+					TerraformVersion: c.TerraformVersion,
+				}
+				projects = append(projects, p)
+
+				profiles[c.Backend.Profile] = AWSRole{
+					AccountID: *c.Backend.AccountID,
+					RoleName:  c.Atlantis.RoleName,
+					RolePath:  c.Atlantis.RolePath,
+				}
+
+				if c.Providers.AWS != nil {
+					a := *c.Providers.AWS
+					profiles[a.Profile] = AWSRole{
+						AccountID: a.AccountID.String(),
+						RoleName:  c.Atlantis.RoleName,
+						RolePath:  c.Atlantis.RolePath,
+					}
+				}
+			}
+		}
+	}
+
+	// sort so that we get stable output
+	sort.SliceStable(projects, func(i, j int) bool {
+		return projects[i].Name < projects[j].Name
+	})
+
+	return Atlantis{
+		Enabled:     enabled,
+		Projects:    projects,
+		AWSProfiles: profiles,
+	}
+}