From ef4ed46c232b7a33ccf4c54b340d366727baf055 Mon Sep 17 00:00:00 2001 From: Ryan King Date: Wed, 27 Jun 2018 10:27:29 -0700 Subject: [PATCH 1/5] global plan --- plan/plan.go | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/plan/plan.go b/plan/plan.go index e4d6719a3..0b091accc 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -64,6 +64,7 @@ type Env struct { type Plan struct { Accounts map[string]account Envs map[string]Env + Global component Modules map[string]module Version string } @@ -83,6 +84,7 @@ func Eval(fs afero.Fs, configFile string) (*Plan, error) { p.Version = v p.Accounts = buildAccounts(c) p.Envs = buildEnvs(c) + p.Global = buildGlobal(c) p.Modules = buildModules(c) return p, nil } @@ -115,6 +117,20 @@ func Print(p *Plan) error { } + fmt.Println("Global:") + fmt.Printf("\tid: %d\n", p.Global.AccountID) + fmt.Printf("\taws_profile_backend: %v\n", p.Global.AWSProfileBackend) + fmt.Printf("\taws_profile_provider: %v\n", p.Global.AWSProfileProvider) + fmt.Printf("\taws_provider_version: %v\n", p.Global.AWSProviderVersion) + fmt.Printf("\taws_region: %v\n", p.Global.AWSRegion) + fmt.Printf("\taws_regions: %v\n", p.Global.AWSRegions) + fmt.Printf("\tinfra_bucket: %v\n", p.Global.InfraBucket) + fmt.Printf("\tname: %v\n", p.Global.AccountName) + fmt.Printf("\tother_p.Globals: %v\n", p.Global.OtherComponents) + fmt.Printf("\towner: %v\n", p.Global.Owner) + fmt.Printf("\tproject: %v\n", p.Global.Project) + fmt.Printf("\tterraform_version: %v\n", p.Global.TerraformVersion) + fmt.Println("Envs:") for name, env := range p.Envs { @@ -206,6 +222,29 @@ func newEnvPlan() Env { return ep } +func buildGlobal(conf *config.Config) component { + // Global just uses defaults because that's the way sicc works. We should make it directly configurable after transition. + componentPlan := component{} + + // TODO add accountID to defaults + componentPlan.AWSRegion = conf.Defaults.AWSRegion + componentPlan.AWSRegions = conf.Defaults.AWSRegions + + componentPlan.AWSProfileBackend = conf.Defaults.AWSProfileBackend + componentPlan.AWSProfileProvider = conf.Defaults.AWSProfileProvider + componentPlan.AWSProviderVersion = conf.Defaults.AWSProviderVersion + // TODO add AccountID to defaults + // componentPlan.AccountID = conf.Defaults.AccountID + + componentPlan.TerraformVersion = conf.Defaults.TerraformVersion + componentPlan.InfraBucket = conf.Defaults.InfraBucket + componentPlan.Owner = conf.Defaults.Owner + componentPlan.Project = conf.Defaults.Project + + componentPlan.Component = "global" + return componentPlan +} + func buildEnvs(conf *config.Config) map[string]Env { envPlans := make(map[string]Env, len(conf.Envs)) defaults := conf.Defaults @@ -230,7 +269,6 @@ func buildEnvs(conf *config.Config) map[string]Env { for componentName, componentConf := range conf.Envs[envName].Components { componentPlan := component{} - componentPlan.AccountID = resolveOptionalInt(envPlan.AccountID, componentConf.AccountID) componentPlan.AWSRegion = resolveRequired(envPlan.AWSRegion, componentConf.AWSRegion) componentPlan.AWSRegions = resolveStringArray(envPlan.AWSRegions, componentConf.AWSRegions) From da191284f7063489cb5c452b4cf058f0eae9b493 Mon Sep 17 00:00:00 2001 From: Ryan King Date: Wed, 27 Jun 2018 13:01:47 -0700 Subject: [PATCH 2/5] wip --- apply/apply.go | 15 ++++- plan/plan.go | 14 ++--- templates/global/Makefile.tmpl | 70 ++++++++++++++++++++++ templates/global/README.md.touch | 0 templates/global/main.tf.touch | 0 templates/global/outputs.tf.touch | 0 templates/global/sicc.tf.tmpl | 92 +++++++++++++++++++++++++++++ templates/global/variables.tf.touch | 0 templates/templates.go | 2 + 9 files changed, 185 insertions(+), 8 deletions(-) create mode 100644 templates/global/Makefile.tmpl create mode 100644 templates/global/README.md.touch create mode 100644 templates/global/main.tf.touch create mode 100644 templates/global/outputs.tf.touch create mode 100644 templates/global/sicc.tf.tmpl create mode 100644 templates/global/variables.tf.touch diff --git a/apply/apply.go b/apply/apply.go index 5bd24204a..75677ad10 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -38,7 +38,9 @@ func Apply(fs afero.Fs, configFile string, tmp *templates.T) error { return e } - // TODO global + e = applyGlobal(fs, p.Global, &tmp.Global) + + // TODO modules return nil } @@ -47,6 +49,17 @@ func applyRepo(fs afero.Fs, p *plan.Plan, repoBox *packr.Box) error { return applyTree(repoBox, fs, p) } +func applyGlobal(fs afero.Fs, p plan.Component, repoBox *packr.Box) error { + log.Println("global") + util.Dump(p) + path := fmt.Sprintf("%s/global", rootPath) + e := fs.MkdirAll(path, 0755) + if e != nil { + return e + } + return applyTree(repoBox, afero.NewBasePathFs(fs, path), p) +} + func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox *packr.Box) (e error) { for account, accountPlan := range p.Accounts { path := fmt.Sprintf("%s/accounts/%s", rootPath, account) diff --git a/plan/plan.go b/plan/plan.go index 0b091accc..d783cb4d7 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -27,7 +27,7 @@ type module struct { TerraformVersion string } -type component struct { +type Component struct { AccountID *int64 AccountName string AWSProfileBackend string @@ -58,13 +58,13 @@ type Env struct { Project string TerraformVersion string - Components map[string]component + Components map[string]Component } type Plan struct { Accounts map[string]account Envs map[string]Env - Global component + Global Component Modules map[string]module Version string } @@ -218,13 +218,13 @@ func buildModules(c *config.Config) map[string]module { func newEnvPlan() Env { ep := Env{} - ep.Components = make(map[string]component) + ep.Components = make(map[string]Component) return ep } -func buildGlobal(conf *config.Config) component { +func buildGlobal(conf *config.Config) Component { // Global just uses defaults because that's the way sicc works. We should make it directly configurable after transition. - componentPlan := component{} + componentPlan := Component{} // TODO add accountID to defaults componentPlan.AWSRegion = conf.Defaults.AWSRegion @@ -267,7 +267,7 @@ func buildEnvs(conf *config.Config) map[string]Env { envPlan.Project = resolveRequired(defaults.Project, envConf.Project) for componentName, componentConf := range conf.Envs[envName].Components { - componentPlan := component{} + componentPlan := Component{} componentPlan.AWSRegion = resolveRequired(envPlan.AWSRegion, componentConf.AWSRegion) componentPlan.AWSRegions = resolveStringArray(envPlan.AWSRegions, componentConf.AWSRegions) diff --git a/templates/global/Makefile.tmpl b/templates/global/Makefile.tmpl new file mode 100644 index 000000000..1dd08508a --- /dev/null +++ b/templates/global/Makefile.tmpl @@ -0,0 +1,70 @@ +# Auto-generated by sicc. Do not edit +# Make improvements in sicc, so that everyone can benefit. + +TF_VARS := $(patsubst %,-e%,$(filter TF_VAR_%,$(.VARIABLES))) +REPO_ROOT := $(shell git rev-parse --show-toplevel) +REPO_RELATIVE_PATH := $(shell git rev-parse --show-prefix) +# We need to do this because `terraform fmt` recurses into .terraform/modules +# and wont' accept more than one file at a time. +TF=$(wildcard *.tf) + +docker_base = \ + docker run -it --rm -e HOME=/home -v $$HOME/.aws:/home/.aws -v $(REPO_ROOT):/repo \ + -e GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \ + -e TF_PLUGIN_CACHE_DIR="/repo/.terraform.d/plugin-cache" -e TF="$(TF)" \ + -w /repo/$(REPO_RELATIVE_PATH) $(TF_VARS) $$($(REPO_ROOT)/scripts/docker-ssh-mount.sh) +docker_terraform = $(docker_base) hashicorp/terraform:{{ .TerraformVersion }} +docker_sh = $(docker_base) --entrypoint='/bin/sh' hashicorp/terraform:{{ .TerraformVersion }} + +all: + +fmt: + @$(docker_sh) -c "for f in $(TF); do printf '.'; terraform fmt $$f; done"; \ + echo + +lint: lint-tf + +lint-tf: + @$(docker_sh) -c "for f in $(TF); do printf '.'; terraform fmt --check=true --diff=true $$f || exit $$? ; done"; \ + echo + +get: ssh-forward + $(docker_terraform) get --update=true + +plan: fmt get init ssh-forward + $(docker_terraform) plan + +apply: fmt get init ssh-forward + $(docker_terraform) apply -auto-approve=false + +docs: + @echo + +clean: + -rm -rfv .terraform/modules + -rm -rfv .terraform/plugins + +test: + +init: ssh-forward + $(docker_terraform) init -input=false + +check-plan: init get ssh-forward + $(docker_terraform) plan -detailed-exitcode; \ + ERR=$$?; \ + if [ $$ERR -eq 0 ] ; then \ + echo "Success"; \ + elif [ $$ERR -eq 1 ] ; then \ + echo "Error in plan execution."; \ + exit 1; \ + elif [ $$ERR -eq 2 ] ; then \ + echo "Diff"; \ + fi + +ssh-forward: + $(REPO_ROOT)/scripts/docker-ssh-forward.sh + +run: + $(docker_terraform) $(CMD) + +.PHONY: all apply clean docs fmt get lint plan run ssh-forward test \ No newline at end of file diff --git a/templates/global/README.md.touch b/templates/global/README.md.touch new file mode 100644 index 000000000..e69de29bb diff --git a/templates/global/main.tf.touch b/templates/global/main.tf.touch new file mode 100644 index 000000000..e69de29bb diff --git a/templates/global/outputs.tf.touch b/templates/global/outputs.tf.touch new file mode 100644 index 000000000..e69de29bb diff --git a/templates/global/sicc.tf.tmpl b/templates/global/sicc.tf.tmpl new file mode 100644 index 000000000..2261e1ede --- /dev/null +++ b/templates/global/sicc.tf.tmpl @@ -0,0 +1,92 @@ +# Auto-generated by sicc. Do not edit +# Make improvements in sicc, so that everyone can benefit. + +provider "aws" { + version = "~> {{ .AWSProviderVersion }}" + region = "{{ .AWSRegion }}" + profile = "{{ .AWSProfileProvider }}" +} + +# Aliased Providers (for doing things in every region). +{{ $out := . }} +{{ range $region := .AWSRegions }} + provider "aws" { + alias = "{{ $region }}" + version = "~> {{ $out.AWSProviderVersion }}" + region = "{{ $region }}" + profile = "{{ $out.AWSProfileProvider }}" + } +{{ end }} + +terraform { + required_version = "~>{{ .TerraformVersion }}" + + backend "s3" { + bucket = "{{ .InfraBucket }}" + {{/* {%- if env is defined and component_name is defined %} */}} + key = "terraform/{{ .Project }}/envs/{{ .Env }}/components/{{ .Component }}.tfstate" + +{{/* + {%- else %} + key = "terraform/{{ project }}/global.tfstate" + {%- endif %} +*/}} + encrypt = true + region = "{{ .AWSRegion }}" + profile = "{{ .AWSProfileBackend }}" + } +} + +variable "env" { + type = "string" + default = "{{ .Env }}" +} + +variable "project" { + type = "string" + default = "{{ .Project }}" +} + +variable "region" { + type = "string" + default = "{{ .AWSRegion }}" +} + +variable "component" { + type = "string" + default = "{{ .Component }}" +} + +variable "aws_profile" { + type = "string" + default = "{{ .AWSProfileProvider }}" +} + +variable "owner" { + type = "string" + default = "{{ .Owner }}" +} + +variable "tags" { + type = "map" + default = { + project = "{{ .Project }}" + env = "{{ .Env }}" + service = "{{ .Component }}" + owner = "{{ .Owner }}" + managedBy = "terraform" + } +} + +{{ range $component := .OtherComponents | sortAlpha }} +data "terraform_remote_state" "{{ $component }}" { + backend = "s3" + + config { + bucket = "{{ $out.InfraBucket }}" + key = "terraform/{{ $out.Project }}/envs/{{ $out.Env }}/components/{{ $component }}.tfstate" + region = "{{ $out.AWSRegion }}" + {{ if $out.AWSProfileBackend }}profile = "{{ $out.AWSProfileBackend }}"{{ end }} + } +} +{{ end }} diff --git a/templates/global/variables.tf.touch b/templates/global/variables.tf.touch new file mode 100644 index 000000000..e69de29bb diff --git a/templates/templates.go b/templates/templates.go index d510f2d44..a2d6b69f9 100644 --- a/templates/templates.go +++ b/templates/templates.go @@ -6,6 +6,7 @@ type T struct { Account packr.Box Component packr.Box Env packr.Box + Global packr.Box Repo packr.Box } @@ -13,5 +14,6 @@ var Templates = &T{ Account: packr.NewBox("account"), Component: packr.NewBox("component"), Env: packr.NewBox("env"), + Global: packr.NewBox("global"), Repo: packr.NewBox("repo"), } From e582502bc008dcadd9329158492774e3b14d7d43 Mon Sep 17 00:00:00 2001 From: Ryan King Date: Wed, 27 Jun 2018 13:20:47 -0700 Subject: [PATCH 3/5] remove debug output --- apply/apply.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/apply/apply.go b/apply/apply.go index 75677ad10..d269218c3 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -50,8 +50,6 @@ func applyRepo(fs afero.Fs, p *plan.Plan, repoBox *packr.Box) error { } func applyGlobal(fs afero.Fs, p plan.Component, repoBox *packr.Box) error { - log.Println("global") - util.Dump(p) path := fmt.Sprintf("%s/global", rootPath) e := fs.MkdirAll(path, 0755) if e != nil { From 2379446723037de6094f772a03637ee35546cfa1 Mon Sep 17 00:00:00 2001 From: Ryan King Date: Thu, 28 Jun 2018 09:16:36 -0700 Subject: [PATCH 4/5] handle error from applyGlobal --- apply/apply.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apply/apply.go b/apply/apply.go index d269218c3..b00016888 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -39,6 +39,9 @@ func Apply(fs afero.Fs, configFile string, tmp *templates.T) error { } e = applyGlobal(fs, p.Global, &tmp.Global) + if e != nil { + return e + } // TODO modules From 6b684b49249342b8e7f035e76274f8c31d1dd306 Mon Sep 17 00:00:00 2001 From: Ryan King Date: Thu, 28 Jun 2018 09:54:07 -0700 Subject: [PATCH 5/5] apply for modules --- apply/apply.go | 21 ++++++++++++- plan/plan.go | 10 +++---- templates/module/Makefile.tmpl | 46 +++++++++++++++++++++++++++++ templates/module/README.md.create | 2 ++ templates/module/main.tf.touch | 0 templates/module/outputs.tf.touch | 0 templates/module/sicc.tf.tmpl | 6 ++++ templates/module/variables.tf.touch | 0 templates/templates.go | 2 ++ 9 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 templates/module/Makefile.tmpl create mode 100644 templates/module/README.md.create create mode 100644 templates/module/main.tf.touch create mode 100644 templates/module/outputs.tf.touch create mode 100644 templates/module/sicc.tf.tmpl create mode 100644 templates/module/variables.tf.touch diff --git a/apply/apply.go b/apply/apply.go index b00016888..553ce2e4e 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -43,7 +43,10 @@ func Apply(fs afero.Fs, configFile string, tmp *templates.T) error { return e } - // TODO modules + e = applyModules(fs, p.Modules, &tmp.Module) + if e != nil { + return e + } return nil } @@ -76,6 +79,22 @@ func applyAccounts(fs afero.Fs, p *plan.Plan, accountBox *packr.Box) (e error) { return nil } +func applyModules(fs afero.Fs, p map[string]plan.Module, moduleBox *packr.Box) error { + var e error + for module, modulePlan := range p { + path := fmt.Sprintf("%s/modules/%s", rootPath, module) + e = fs.MkdirAll(path, 0755) + if e != nil { + return e + } + e = applyTree(moduleBox, afero.NewBasePathFs(fs, path), modulePlan) + if e != nil { + return e + } + } + return nil +} + func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBox *packr.Box) (e error) { for env, envPlan := range p.Envs { path := fmt.Sprintf("%s/envs/%s", rootPath, env) diff --git a/plan/plan.go b/plan/plan.go index d783cb4d7..ed4a06eee 100644 --- a/plan/plan.go +++ b/plan/plan.go @@ -23,7 +23,7 @@ type account struct { TerraformVersion string } -type module struct { +type Module struct { TerraformVersion string } @@ -65,7 +65,7 @@ type Plan struct { Accounts map[string]account Envs map[string]Env Global Component - Modules map[string]module + Modules map[string]Module Version string } @@ -205,10 +205,10 @@ func buildAccounts(c *config.Config) map[string]account { return accountPlans } -func buildModules(c *config.Config) map[string]module { - modulePlans := make(map[string]module, len(c.Modules)) +func buildModules(c *config.Config) map[string]Module { + modulePlans := make(map[string]Module, len(c.Modules)) for name, conf := range c.Modules { - modulePlan := module{} + modulePlan := Module{} modulePlan.TerraformVersion = resolveRequired(c.Defaults.TerraformVersion, conf.TerraformVersion) modulePlans[name] = modulePlan diff --git a/templates/module/Makefile.tmpl b/templates/module/Makefile.tmpl new file mode 100644 index 000000000..1612d61d7 --- /dev/null +++ b/templates/module/Makefile.tmpl @@ -0,0 +1,46 @@ +# Auto-generated by sicc. Do not edit +# Make improvements in sicc, so that everyone can benefit. + +TF_VARS := $(patsubst %,-e%,$(filter TF_VAR_%,$(.VARIABLES))) +REPO_ROOT := $(shell git rev-parse --show-toplevel) +REPO_RELATIVE_PATH := $(shell git rev-parse --show-prefix) +# We need to do this because `terraform fmt` recurses into .terraform/modules +# and wont' accept more than one file at a time. +TF=$(wildcard *.tf) + +docker_base = \ + docker run -it --rm -e HOME=/home -v $$HOME/.aws:/home/.aws -v $(REPO_ROOT):/repo \ + -e GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \ + -e TF_PLUGIN_CACHE_DIR="/repo/.terraform.d/plugin-cache" -e TF="$(TF)" \ + -w /repo/$(REPO_RELATIVE_PATH) $(TF_VARS) $$($(REPO_ROOT)/scripts/docker-ssh-mount.sh) +docker_terraform = $(docker_base) hashicorp/terraform:{{ .TerraformVersion }} +docker_sh = $(docker_base) --entrypoint='/bin/sh' hashicorp/terraform:{{ .TerraformVersion }} + +all: fmt lint doc + +fmt: + @$(docker_sh) -c "for f in $(TF); do printf '.'; terraform fmt $$f; done"; \ + echo + +lint: lint-tf + +lint-tf: + @$(docker_sh) -c "for f in $(TF); do printf '.'; terraform fmt --check=true --diff=true $$f || exit $$? ; done"; \ + echo + +readme: + bash .update-readme.sh update + +docs: readme + +check-docs: + @bash .update-readme.sh check; \ + if [ ! $$? -eq 0 ]; then \ + echo "Docs are out of date, run \`make docs\`"; \ + fi + +clean: + +test: + +.PHONY: all check-doc clean docs fmt lint lint-tf readme test \ No newline at end of file diff --git a/templates/module/README.md.create b/templates/module/README.md.create new file mode 100644 index 000000000..fce2ac06f --- /dev/null +++ b/templates/module/README.md.create @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/templates/module/main.tf.touch b/templates/module/main.tf.touch new file mode 100644 index 000000000..e69de29bb diff --git a/templates/module/outputs.tf.touch b/templates/module/outputs.tf.touch new file mode 100644 index 000000000..e69de29bb diff --git a/templates/module/sicc.tf.tmpl b/templates/module/sicc.tf.tmpl new file mode 100644 index 000000000..902c28a52 --- /dev/null +++ b/templates/module/sicc.tf.tmpl @@ -0,0 +1,6 @@ +# Auto-generated by sicc. Do not edit +# Make improvements in sicc, so that everyone can benefit. + +terraform { + required_version = "~>{{ .TerraformVersion }}" +} \ No newline at end of file diff --git a/templates/module/variables.tf.touch b/templates/module/variables.tf.touch new file mode 100644 index 000000000..e69de29bb diff --git a/templates/templates.go b/templates/templates.go index a2d6b69f9..23189933a 100644 --- a/templates/templates.go +++ b/templates/templates.go @@ -7,6 +7,7 @@ type T struct { Component packr.Box Env packr.Box Global packr.Box + Module packr.Box Repo packr.Box } @@ -15,5 +16,6 @@ var Templates = &T{ Component: packr.NewBox("component"), Env: packr.NewBox("env"), Global: packr.NewBox("global"), + Module: packr.NewBox("module"), Repo: packr.NewBox("repo"), }