From f2595ff86558b62952f56b72b9c0def42642befa Mon Sep 17 00:00:00 2001 From: Andrii Nasinnyk Date: Fri, 31 May 2019 01:02:15 +0300 Subject: [PATCH] feat: added var step for write command to env variable runatlantis/atlantis#370 Signed-off-by: Andrii Nasinnyk --- server/events/models/models.go | 2 + server/events/project_command_runner.go | 2 + server/events/runtime/apply_step_runner.go | 4 +- server/events/runtime/init_step_runner.go | 2 +- server/events/runtime/plan_step_runner.go | 10 +-- server/events/runtime/run_step_runner.go | 3 + server/events/terraform/terraform_client.go | 14 +++- server/events/yaml/raw/step.go | 81 +++++++++++++++++++-- server/events/yaml/raw/step_test.go | 29 ++++++++ server/events/yaml/valid/repo_cfg.go | 1 + 10 files changed, 133 insertions(+), 15 deletions(-) diff --git a/server/events/models/models.go b/server/events/models/models.go index 2fe503f534..0ae8abb788 100644 --- a/server/events/models/models.go +++ b/server/events/models/models.go @@ -343,6 +343,8 @@ type ProjectCommandContext struct { // Workspace is the Terraform workspace this project is in. It will always // be set. Workspace string + // Additional env variables + Env map[string]string } // SplitRepoFullName splits a repo full name up into its owner and repo name diff --git a/server/events/project_command_runner.go b/server/events/project_command_runner.go index cde4769497..3417d6a15d 100644 --- a/server/events/project_command_runner.go +++ b/server/events/project_command_runner.go @@ -185,6 +185,8 @@ func (p *DefaultProjectCommandRunner) runSteps(steps []valid.Step, ctx models.Pr out, err = p.ApplyStepRunner.Run(ctx, step.ExtraArgs, absPath) case "run": out, err = p.RunStepRunner.Run(ctx, step.RunCommand, absPath) + case "var": + ctx.Env[step.Variable], err = p.RunStepRunner.Run(ctx, step.RunCommand, absPath) } if out != "" { diff --git a/server/events/runtime/apply_step_runner.go b/server/events/runtime/apply_step_runner.go index 1fbf61bbb0..86e0af5e60 100644 --- a/server/events/runtime/apply_step_runner.go +++ b/server/events/runtime/apply_step_runner.go @@ -47,7 +47,7 @@ func (a *ApplyStepRunner) Run(ctx models.ProjectCommandContext, extraArgs []stri // NOTE: we need to quote the plan path because Bitbucket Server can // have spaces in its repo owner names which is part of the path. args := append(append(append([]string{"apply", "-input=false", "-no-color"}, extraArgs...), ctx.CommentArgs...), fmt.Sprintf("%q", planPath)) - out, err = a.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, args, ctx.TerraformVersion, ctx.Workspace) + out, err = a.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, args, ctx.Env, ctx.TerraformVersion, ctx.Workspace) } // If the apply was successful, delete the plan. @@ -138,7 +138,7 @@ func (a *ApplyStepRunner) runRemoteApply( // Start the async command execution. ctx.Log.Debug("starting async tf remote operation") - inCh, outCh := a.AsyncTFExec.RunCommandAsync(ctx.Log, filepath.Clean(path), applyArgs, tfVersion, ctx.Workspace) + inCh, outCh := a.AsyncTFExec.RunCommandAsync(ctx.Log, filepath.Clean(path), applyArgs, ctx.Env, tfVersion, ctx.Workspace) var lines []string nextLineIsRunURL := false var runURL string diff --git a/server/events/runtime/init_step_runner.go b/server/events/runtime/init_step_runner.go index 923f3ea1fd..ea958d3d39 100644 --- a/server/events/runtime/init_step_runner.go +++ b/server/events/runtime/init_step_runner.go @@ -24,7 +24,7 @@ func (i *InitStepRunner) Run(ctx models.ProjectCommandContext, extraArgs []strin terraformInitCmd = append([]string{"get", "-no-color", "-upgrade"}, extraArgs...) } - out, err := i.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, terraformInitCmd, tfVersion, ctx.Workspace) + out, err := i.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, terraformInitCmd, ctx.Env, tfVersion, ctx.Workspace) // Only include the init output if there was an error. Otherwise it's // unnecessary and lengthens the comment. if err != nil { diff --git a/server/events/runtime/plan_step_runner.go b/server/events/runtime/plan_step_runner.go index 43e6b44208..bfca0f01f8 100644 --- a/server/events/runtime/plan_step_runner.go +++ b/server/events/runtime/plan_step_runner.go @@ -47,7 +47,7 @@ func (p *PlanStepRunner) Run(ctx models.ProjectCommandContext, extraArgs []strin planFile := filepath.Join(path, GetPlanFilename(ctx.Workspace, ctx.ProjectName)) planCmd := p.buildPlanCmd(ctx, extraArgs, path, tfVersion, planFile) - output, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, filepath.Clean(path), planCmd, tfVersion, ctx.Workspace) + output, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, filepath.Clean(path), planCmd, ctx.Env, tfVersion, ctx.Workspace) if p.isRemoteOpsErr(output, err) { ctx.Log.Debug("detected that this project is using TFE remote ops") return p.remotePlan(ctx, extraArgs, path, tfVersion, planFile) @@ -130,7 +130,7 @@ func (p *PlanStepRunner) switchWorkspace(ctx models.ProjectCommandContext, path // already in the right workspace then no need to switch. This will save us // about ten seconds. This command is only available in > 0.10. if !runningZeroPointNine { - workspaceShowOutput, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "show"}, tfVersion, ctx.Workspace) + workspaceShowOutput, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "show"}, ctx.Env, tfVersion, ctx.Workspace) if err != nil { return err } @@ -145,11 +145,11 @@ func (p *PlanStepRunner) switchWorkspace(ctx models.ProjectCommandContext, path // To do this we can either select and catch the error or use list and then // look for the workspace. Both commands take the same amount of time so // that's why we're running select here. - _, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "select", "-no-color", ctx.Workspace}, tfVersion, ctx.Workspace) + _, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "select", "-no-color", ctx.Workspace}, ctx.Env, tfVersion, ctx.Workspace) if err != nil { // If terraform workspace select fails we run terraform workspace // new to create a new workspace automatically. - _, err = p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "new", "-no-color", ctx.Workspace}, tfVersion, ctx.Workspace) + _, err = p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "new", "-no-color", ctx.Workspace}, ctx.Env, tfVersion, ctx.Workspace) return err } return nil @@ -261,7 +261,7 @@ func (p *PlanStepRunner) runRemotePlan( // Start the async command execution. ctx.Log.Debug("starting async tf remote operation") - _, outCh := p.AsyncTFExec.RunCommandAsync(ctx.Log, filepath.Clean(path), cmdArgs, tfVersion, ctx.Workspace) + _, outCh := p.AsyncTFExec.RunCommandAsync(ctx.Log, filepath.Clean(path), cmdArgs, ctx.Env, tfVersion, ctx.Workspace) var lines []string nextLineIsRunURL := false var runURL string diff --git a/server/events/runtime/run_step_runner.go b/server/events/runtime/run_step_runner.go index f7a7cb2fce..64c9bfda18 100644 --- a/server/events/runtime/run_step_runner.go +++ b/server/events/runtime/run_step_runner.go @@ -44,6 +44,9 @@ func (r *RunStepRunner) Run(ctx models.ProjectCommandContext, command string, pa for key, val := range customEnvVars { finalEnvVars = append(finalEnvVars, fmt.Sprintf("%s=%s", key, val)) } + for key, val := range ctx.Env { + finalEnvVars = append(finalEnvVars, fmt.Sprintf("%s=%s", key, val)) + } cmd.Env = finalEnvVars out, err := cmd.CombinedOutput() diff --git a/server/events/terraform/terraform_client.go b/server/events/terraform/terraform_client.go index a42d7bd825..36840e5154 100644 --- a/server/events/terraform/terraform_client.go +++ b/server/events/terraform/terraform_client.go @@ -178,11 +178,16 @@ func (c *DefaultClient) DefaultVersion() *version.Version { } // See Client.RunCommandWithVersion. -func (c *DefaultClient) RunCommandWithVersion(log *logging.SimpleLogger, path string, args []string, v *version.Version, workspace string) (string, error) { +func (c *DefaultClient) RunCommandWithVersion(log *logging.SimpleLogger, path string, args []string, customEnvVars map[string]string, v *version.Version, workspace string) (string, error) { tfCmd, cmd, err := c.prepCmd(log, v, workspace, path, args) if err != nil { return "", err } + envVars := cmd.Env + for key, val := range customEnvVars { + envVars = append(envVars, fmt.Sprintf("%s=%s", key, val)) + } + cmd.Env = envVars out, err := cmd.CombinedOutput() if err != nil { err = errors.Wrapf(err, "running %q in %q", tfCmd, path) @@ -250,7 +255,7 @@ type Line struct { // Callers can use the input channel to pass stdin input to the command. // If any error is passed on the out channel, there will be no // further output (so callers are free to exit). -func (c *DefaultClient) RunCommandAsync(log *logging.SimpleLogger, path string, args []string, v *version.Version, workspace string) (chan<- string, <-chan Line) { +func (c *DefaultClient) RunCommandAsync(log *logging.SimpleLogger, path string, args []string, customEnvVars map[string]string, v *version.Version, workspace string) (chan<- string, <-chan Line) { outCh := make(chan Line) inCh := make(chan string) @@ -273,6 +278,11 @@ func (c *DefaultClient) RunCommandAsync(log *logging.SimpleLogger, path string, stdout, _ := cmd.StdoutPipe() stderr, _ := cmd.StderrPipe() stdin, _ := cmd.StdinPipe() + envVars := cmd.Env + for key, val := range customEnvVars { + envVars = append(envVars, fmt.Sprintf("%s=%s", key, val)) + } + cmd.Env = envVars log.Debug("starting %q in %q", tfCmd, path) err = cmd.Start() diff --git a/server/events/yaml/raw/step.go b/server/events/yaml/raw/step.go index 12e22a5d6a..2a30bd0e35 100644 --- a/server/events/yaml/raw/step.go +++ b/server/events/yaml/raw/step.go @@ -13,20 +13,27 @@ import ( const ( ExtraArgsKey = "extra_args" + NameArgKey = "name" + CommandArgKey = "command" RunStepName = "run" PlanStepName = "plan" ApplyStepName = "apply" InitStepName = "init" + VarStepName = "var" ) // Step represents a single action/command to perform. In YAML, it can be set as // 1. A single string for a built-in command: // - init // - plan -// 2. A map for a built-in command and extra_args: +// 2. A map for a built-in var with name and command +// - var: +// name: test +// command: echo 312 +// 3. A map for a built-in command and extra_args: // - plan: // extra_args: [-var-file=staging.tfvars] -// 3. A map for a custom run command: +// 4. A map for a custom run command: // - run: my custom command // Here we parse step in the most generic fashion possible. See fields for more // details. @@ -35,8 +42,10 @@ type Step struct { // could be multiple keys (since the element is a map) so we don't set Key. Key *string // Map will be set in case #2 above. + Var map[string]map[string]string + // Map will be set in case #3 above. Map map[string]map[string][]string - // StringVal will be set in case #3 above. + // StringVal will be set in case #4 above. StringVal map[string]string } @@ -52,7 +61,7 @@ func (s *Step) UnmarshalJSON(data []byte) error { func (s Step) Validate() error { validStep := func(value interface{}) error { str := *value.(*string) - if str != InitStepName && str != PlanStepName && str != ApplyStepName { + if str != InitStepName && str != PlanStepName && str != ApplyStepName && str != VarStepName { return fmt.Errorf("%q is not a valid step type, maybe you omitted the 'run' key", str) } return nil @@ -94,6 +103,40 @@ func (s Step) Validate() error { return nil } + varStep := func(value interface{}) error { + elem := value.(map[string]map[string]string) + var keys []string + for k := range elem { + keys = append(keys, k) + } + // Sort so tests can be deterministic. + sort.Strings(keys) + + if len(keys) > 1 { + return fmt.Errorf("step element can only contain a single key, found %d: %s", + len(keys), strings.Join(keys, ",")) + } + for stepName, args := range elem { + if stepName != VarStepName { + return fmt.Errorf("%q is not a valid step type", stepName) + } + var argKeys []string + for k := range args { + argKeys = append(argKeys, k) + } + if len(argKeys) != 2 { + return fmt.Errorf("built-in steps only support two keys %s and %s, found %d: %s", + NameArgKey, CommandArgKey, len(argKeys), strings.Join(argKeys, ",")) + } + for k := range args { + if k != NameArgKey && k != CommandArgKey { + return fmt.Errorf("built-in steps only support two keys %s and %s, found %q in step %s", NameArgKey, CommandArgKey, k, stepName) + } + } + } + return nil + } + runStep := func(value interface{}) error { elem := value.(map[string]string) var keys []string @@ -121,6 +164,9 @@ func (s Step) Validate() error { if len(s.Map) > 0 { return validation.Validate(s.Map, validation.By(extraArgs)) } + if len(s.Var) > 0 { + return validation.Validate(s.Var, validation.By(varStep)) + } if len(s.StringVal) > 0 { return validation.Validate(s.StringVal, validation.By(runStep)) } @@ -136,6 +182,19 @@ func (s Step) ToValid() valid.Step { } // This will trigger in case #2 (see Step docs). + if len(s.Var) > 0 { + // After validation we assume there's only one key and it's a valid + // step name so we just use the first one. + for stepName, stepArgs := range s.Var { + return valid.Step{ + StepName: stepName, + Variable: stepArgs[NameArgKey], + RunCommand: stepArgs[CommandArgKey], + } + } + } + + // This will trigger in case #3 (see Step docs). if len(s.Map) > 0 { // After validation we assume there's only one key and it's a valid // step name so we just use the first one. @@ -147,7 +206,7 @@ func (s Step) ToValid() valid.Step { } } - // This will trigger in case #3 (see Step docs). + // This will trigger in case #4 (see Step docs). if len(s.StringVal) > 0 { // After validation we assume there's only one key and it's a valid // step name so we just use the first one. @@ -196,6 +255,18 @@ func (s *Step) unmarshalGeneric(unmarshal func(interface{}) error) error { return nil } + // This represents a step with extra_args, ex: + // var: + // name: k + // command: exec + // We validate if the key var + var varStep map[string]map[string]string + err = unmarshal(&varStep) + if err == nil { + s.Var = varStep + return nil + } + // Try to unmarshal as a custom run step, ex. // steps: // - run: my command diff --git a/server/events/yaml/raw/step_test.go b/server/events/yaml/raw/step_test.go index fd0cbe3c93..2fc99df224 100644 --- a/server/events/yaml/raw/step_test.go +++ b/server/events/yaml/raw/step_test.go @@ -184,6 +184,18 @@ func TestStep_Validate(t *testing.T) { }, expErr: "", }, + { + description: "var", + input: raw.Step{ + Var: VarType{ + "var": { + "name": "test", + "command": "echo 123", + }, + }, + }, + expErr: "", + }, { description: "apply extra_args", input: raw.Step{ @@ -323,6 +335,22 @@ func TestStep_ToValid(t *testing.T) { StepName: "apply", }, }, + { + description: "var step", + input: raw.Step{ + Var: VarType{ + "var": { + "name": "test", + "command": "echo 123", + }, + }, + }, + exp: valid.Step{ + StepName: "var", + RunCommand: "echo 123", + Variable: "test", + }, + }, { description: "init extra_args", input: raw.Step{ @@ -386,3 +414,4 @@ func TestStep_ToValid(t *testing.T) { } type MapType map[string]map[string][]string +type VarType map[string]map[string]string diff --git a/server/events/yaml/valid/repo_cfg.go b/server/events/yaml/valid/repo_cfg.go index e5aaa4c136..2d5bab5855 100644 --- a/server/events/yaml/valid/repo_cfg.go +++ b/server/events/yaml/valid/repo_cfg.go @@ -75,6 +75,7 @@ type Step struct { StepName string ExtraArgs []string RunCommand string + Variable string } type Workflow struct {