feat: set the shell for workflow command steps

Signed-off-by: anryko <[email protected]>

runatlantis.io/docs/custom-workflows.md

@@ -599,13 +599,15 @@ Full
 ```yaml
 - run:
     command: custom-command arg1 arg2
+    shell: sh
     output: show
 ```
 
 | Key | Type                                                         | Default | Required | Description                                                                                                                                                                                                                                                                                                                                                                                             |
 |-----|--------------------------------------------------------------|---------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | run | map\[string -> string\] | none    | no       | Run a custom command                                                                                                                                                                                                                                                                                                                                                                                    |
 | run.command | string                                                       | none | yes      | Shell command to run                                                                                                                                                                                                                                                                                                                                                                                    |
+| run.shell   | string | "sh" | no | Name of the shell to use for command execution (valid values are "sh" and "bash") |                                                    |
 | run.output | string                                                       | "show" | no       | How to post-process the output of this command when posted in the PR comment. The options are<br/>*`show` - preserve the full output<br/>* `hide` - hide output from comment (still visible in the real-time streaming output)<br/> * `strip_refreshing` - hide all output up until and including the last line containing "Refreshing...". This matches the behavior of the built-in `plan` command |
 
 #### Native Environment Variables
@@ -664,6 +666,10 @@ as the environment variable value.
 - env:
     name: ENV_NAME_2
     command: 'echo "dynamic-value-$(date)"'
+- env:
+    name: ENV_NAME_3
+    command: echo ${DIR%$REPO_REL_DIR}
+    shell: bash
 ```
 
 | Key             | Type                  | Default | Required | Description                                                                                                     |
@@ -672,6 +678,7 @@ as the environment variable value.
 | env.name | string | none | yes | Name of the environment variable                                                                                |
 | env.value | string | none | no | Set the value of the environment variable to a hard-coded string. Cannot be set at the same time as `command`   |
 | env.command | string | none | no | Set the value of the environment variable to the output of a command. Cannot be set at the same time as `value` |
+| env.shell   | string | "sh" | no | Name of the shell to use for command execution (valid values are "sh" and "bash"). Cannot be set without `command`|                                                    |
 
 ::: tip Notes
 
@@ -699,13 +706,15 @@ Full:
 ```yaml
 - multienv:
     command: custom-command
+    shell: bash
     output: show
 ```
 
 | Key              | Type                  | Default | Required | Description                                                                         |
 |------------------|-----------------------|---------|----------|-------------------------------------------------------------------------------------|
 | multienv         | map[string -> string] | none    | no       | Run a custom command and add printed environment variables                          |
 | multienv.command | string                | none    | yes      | Name of the custom script to run                                                    |
+| multienv.shell   | string                | "sh"    | no       | Name of the shell to use for command execution (valid values are "sh" and "bash")   |                                                    |
 | multienv.output  | string                | "show"  | no       | Setting output to "hide" will supress the message obout added environment variables |
 
 The output of the command execution must have the following format:

server/core/config/raw/step.go

@@ -9,6 +9,7 @@ import (
 
 	validation "github.com/go-ozzo/ozzo-validation"
 	"github.com/runatlantis/atlantis/server/core/config/valid"
+	"github.com/runatlantis/atlantis/server/utils"
 )
 
 const (
@@ -27,35 +28,44 @@ const (
 	MultiEnvStepName    = "multienv"
 	ImportStepName      = "import"
 	StateRmStepName     = "state_rm"
+	ShellArgKey         = "shell"
 )
 
-// Step represents a single action/command to perform. In YAML, it can be set as
-// 1. A single string for a built-in command:
-//   - init
-//   - plan
-//   - policy_check
-//
-// 2. A map for an env step with name and command or value, or a run step with a command and output config
-//   - env:
-//     name: test
-//     command: echo 312
-//     value: value
-//   - multienv:
-//     command: envs.sh
-//     outpiut: hide
-//   - run:
-//     command: my custom command
-//     output: hide
-//
-// 3. A map for a built-in command and extra_args:
-//   - plan:
-//     extra_args: [-var-file=staging.tfvars]
-//
-// 4. A map for a custom run command:
-//   - run: my custom command
-//
-// Here we parse step in the most generic fashion possible. See fields for more
-// details.
+/*
+Step represents a single action/command to perform. In YAML, it can be set as
+1. A single string for a built-in command:
+  - init
+  - plan
+  - policy_check
+
+2. A map for an env step with name and command or value, or a run step with a command and output config
+  - env:
+    name: test_command
+    command: echo 312
+  - env:
+    name: test_value
+    value: value
+  - env:
+    name: test_bash_command
+    command: echo ${test_value::7}
+    shell: bash
+  - multienv:
+    command: envs.sh
+    output: hide
+  - run:
+    command: my custom command
+    output: hide
+
+3. A map for a built-in command and extra_args:
+  - plan:
+    extra_args: [-var-file=staging.tfvars]
+
+4. A map for a custom run command:
+  - run: my custom command
+
+Here we parse step in the most generic fashion possible. See fields for more
+details.
+*/
 type Step struct {
 	// Key will be set in case #1 and #3 above to the key. In case #2, there
 	// could be multiple keys (since the element is a map) so we don't set Key.
@@ -180,8 +190,9 @@ func (s Step) Validate() error {
 
 			foundNameKey := false
 			for _, k := range argKeys {
-				if k != NameArgKey && k != CommandArgKey && k != ValueArgKey {
-					return fmt.Errorf("env steps only support keys %q, %q and %q, found key %q", NameArgKey, ValueArgKey, CommandArgKey, k)
+				if k != NameArgKey && k != CommandArgKey && k != ValueArgKey && k != ShellArgKey {
+					return fmt.Errorf("env steps only support keys %q, %q, %q and %q, found key %q",
+						NameArgKey, ValueArgKey, CommandArgKey, ShellArgKey, k)
 				}
 				if k == NameArgKey {
 					foundNameKey = true
@@ -190,11 +201,14 @@ func (s Step) Validate() error {
 			if !foundNameKey {
 				return fmt.Errorf("env steps must have a %q key set", NameArgKey)
 			}
-			// If we have 3 keys at this point then they've set both command and value.
-			if len(argKeys) != 2 {
+			if utils.SlicesContains(argKeys, ValueArgKey) && utils.SlicesContains(argKeys, CommandArgKey) {
 				return fmt.Errorf("env steps only support one of the %q or %q keys, found both",
 					ValueArgKey, CommandArgKey)
 			}
+			if utils.SlicesContains(argKeys, ShellArgKey) && !utils.SlicesContains(argKeys, CommandArgKey) {
+				return fmt.Errorf("env steps only support %q key in combination with %q key",
+					ShellArgKey, CommandArgKey)
+			}
 		case RunStepName, MultiEnvStepName:
 			argsCopy := make(map[string]string)
 			for k, v := range args {
@@ -206,21 +220,34 @@ func (s Step) Validate() error {
 			}
 			delete(args, CommandArgKey)
 			if v, ok := args[OutputArgKey]; ok {
-				if stepName == RunStepName && !(v == valid.PostProcessRunOutputShow || v == valid.PostProcessRunOutputHide || v == valid.PostProcessRunOutputStripRefreshing) {
-					return fmt.Errorf("run step %q option must be one of %q, %q, or %q", OutputArgKey, valid.PostProcessRunOutputShow, valid.PostProcessRunOutputHide, valid.PostProcessRunOutputStripRefreshing)
-				} else if stepName == MultiEnvStepName && !(v == valid.PostProcessRunOutputShow || v == valid.PostProcessRunOutputHide) {
-					return fmt.Errorf("multienv step %q option must be %q or %q", OutputArgKey, valid.PostProcessRunOutputShow, valid.PostProcessRunOutputHide)
+				if stepName == RunStepName && !(v == valid.PostProcessRunOutputShow ||
+					v == valid.PostProcessRunOutputHide || v == valid.PostProcessRunOutputStripRefreshing) {
+					return fmt.Errorf("run step %q option must be one of %q, %q, or %q",
+						OutputArgKey, valid.PostProcessRunOutputShow, valid.PostProcessRunOutputHide,
+						valid.PostProcessRunOutputStripRefreshing)
+				} else if stepName == MultiEnvStepName && !(v == valid.PostProcessRunOutputShow ||
+					v == valid.PostProcessRunOutputHide) {
+					return fmt.Errorf("multienv step %q option must be %q or %q",
+						OutputArgKey, valid.PostProcessRunOutputShow, valid.PostProcessRunOutputHide)
 				}
 			}
 			delete(args, OutputArgKey)
+			if v, ok := args[ShellArgKey]; ok {
+				if !utils.SlicesContains(valid.AllowedRunShellValues, v) {
+					return fmt.Errorf("run step %q value %q is not supported, supported values are: [%s]",
+						ShellArgKey, v, strings.Join(valid.AllowedRunShellValues, ", "))
+				}
+			}
+			delete(args, ShellArgKey)
 			if len(args) > 0 {
 				var argKeys []string
 				for k := range args {
 					argKeys = append(argKeys, k)
 				}
 				// Sort so tests can be deterministic.
 				sort.Strings(argKeys)
-				return fmt.Errorf("%q steps only support keys %q and %q, found extra keys %q", stepName, CommandArgKey, OutputArgKey, strings.Join(argKeys, ","))
+				return fmt.Errorf("%q steps only support keys %q, %q and %q, found extra keys %q",
+					stepName, CommandArgKey, OutputArgKey, ShellArgKey, strings.Join(argKeys, ","))
 			}
 		default:
 			return fmt.Errorf("%q is not a valid step type", stepName)
@@ -284,6 +311,7 @@ func (s Step) ToValid() valid.Step {
 				RunCommand:  stepArgs[CommandArgKey],
 				EnvVarValue: stepArgs[ValueArgKey],
 				Output:      valid.PostProcessRunOutputOption(stepArgs[OutputArgKey]),
+				RunShell:    stepArgs[ShellArgKey],
 			}
 			if step.StepName == RunStepName && step.Output == "" {
 				step.Output = valid.PostProcessRunOutputShow

server/core/config/raw/step_test.go

@@ -371,7 +371,7 @@ func TestStep_Validate(t *testing.T) {
 					},
 				},
 			},
-			expErr: "env steps only support keys \"name\", \"value\" and \"command\", found key \"abc\"",
+			expErr: "env steps only support keys \"name\", \"value\", \"command\" and \"shell\", found key \"abc\"",
 		},
 		{
 			description: "env step with both command and value set",
@@ -386,6 +386,19 @@ func TestStep_Validate(t *testing.T) {
 			},
 			expErr: "env steps only support one of the \"value\" or \"command\" keys, found both",
 		},
+		{
+			description: "env step with both shell and value set",
+			input: raw.Step{
+				CommandMap: EnvType{
+					"env": {
+						"name":  "name",
+						"shell": "shell",
+						"value": "value",
+					},
+				},
+			},
+			expErr: "env steps only support \"shell\" key in combination with \"command\" key",
+		},
 		{
 			// For atlantis.yaml v2, this wouldn't parse, but now there should
 			// be no error.

server/core/config/valid/repo_cfg.go

@@ -185,6 +185,8 @@ const (
 	PostProcessRunOutputStripRefreshing = "strip_refreshing"
 )
 
+var AllowedRunShellValues = []string{"sh", "bash"}
+
 type Stage struct {
 	Steps []Step
 }
@@ -202,6 +204,8 @@ type Step struct {
 	EnvVarName string
 	// EnvVarValue is the value to set EnvVarName to.
 	EnvVarValue string
+	// The Shell to use for RunCommand execution.
+	RunShell string
 }
 
 type Workflow struct {

server/core/runtime/env_step_runner.go

@@ -15,13 +15,20 @@ type EnvStepRunner struct {
 // Run runs the env step command.
 // value is the value for the environment variable. If set this is returned as
 // the value. Otherwise command is run and its output is the value returned.
-func (r *EnvStepRunner) Run(ctx command.ProjectContext, command string, value string, path string, envs map[string]string) (string, error) {
+func (r *EnvStepRunner) Run(
+	ctx command.ProjectContext,
+	shell string,
+	command string,
+	value string,
+	path string,
+	envs map[string]string,
+) (string, error) {
 	if value != "" {
 		return value, nil
 	}
 	// Pass `false` for streamOutput because this isn't interesting to the user reading the build logs
 	// in the web UI.
-	res, err := r.RunStepRunner.Run(ctx, command, path, envs, false, valid.PostProcessRunOutputShow)
+	res, err := r.RunStepRunner.Run(ctx, shell, command, path, envs, false, valid.PostProcessRunOutputShow)
 	// Trim newline from res to support running `echo env_value` which has
 	// a newline. We don't recommend users run echo -n env_value to remove the
 	// newline because -n doesn't work in the sh shell which is what we use

server/core/runtime/models/shell_command_runner.go

@@ -16,6 +16,7 @@ import (
 
 // Setting the buffer size to 10mb
 const BufioScannerBufferSize = 10 * 1024 * 1024
+const DefaultRunShell = "sh"
 
 // Line represents a line that was output from a shell command.
 type Line struct {
@@ -35,8 +36,19 @@ type ShellCommandRunner struct {
 	cmd           *exec.Cmd
 }
 
-func NewShellCommandRunner(command string, environ []string, workingDir string, streamOutput bool, outputHandler jobs.ProjectCommandOutputHandler) *ShellCommandRunner {
-	cmd := exec.Command("sh", "-c", command) // #nosec
+func NewShellCommandRunner(
+	shell string,
+	command string,
+	environ []string,
+	workingDir string,
+	streamOutput bool,
+	outputHandler jobs.ProjectCommandOutputHandler,
+) *ShellCommandRunner {
+	if shell == "" {
+		shell = DefaultRunShell
+	}
+
+	cmd := exec.Command(shell, "-c", command) // #nosec
 	cmd.Env = environ
 	cmd.Dir = workingDir
 

server/core/runtime/multienv_step_runner.go

@@ -16,8 +16,15 @@ type MultiEnvStepRunner struct {
 
 // Run runs the multienv step command.
 // The command must return a json string containing the array of name-value pairs that are being added as extra environment variables
-func (r *MultiEnvStepRunner) Run(ctx command.ProjectContext, command string, path string, envs map[string]string, postProcessOutput valid.PostProcessRunOutputOption) (string, error) {
-	res, err := r.RunStepRunner.Run(ctx, command, path, envs, false, postProcessOutput)
+func (r *MultiEnvStepRunner) Run(
+	ctx command.ProjectContext,
+	shell string,
+	command string,
+	path string,
+	envs map[string]string,
+	postProcessOutput valid.PostProcessRunOutputOption,
+) (string, error) {
+	res, err := r.RunStepRunner.Run(ctx, shell, command, path, envs, false, postProcessOutput)
 	if err != nil {
 		return "", err
 	}

server/core/runtime/run_step_runner.go

@@ -22,7 +22,15 @@ type RunStepRunner struct {
 	ProjectCmdOutputHandler jobs.ProjectCommandOutputHandler
 }
 
-func (r *RunStepRunner) Run(ctx command.ProjectContext, command string, path string, envs map[string]string, streamOutput bool, postProcessOutput valid.PostProcessRunOutputOption) (string, error) {
+func (r *RunStepRunner) Run(
+	ctx command.ProjectContext,
+	shell string,
+	command string,
+	path string,
+	envs map[string]string,
+	streamOutput bool,
+	postProcessOutput valid.PostProcessRunOutputOption,
+) (string, error) {
 	tfVersion := r.DefaultTFVersion
 	if ctx.TerraformVersion != nil {
 		tfVersion = ctx.TerraformVersion
@@ -68,7 +76,7 @@ func (r *RunStepRunner) Run(ctx command.ProjectContext, command string, path str
 		finalEnvVars = append(finalEnvVars, fmt.Sprintf("%s=%s", key, val))
 	}
 
-	runner := models.NewShellCommandRunner(command, finalEnvVars, path, streamOutput, r.ProjectCmdOutputHandler)
+	runner := models.NewShellCommandRunner(shell, command, finalEnvVars, path, streamOutput, r.ProjectCmdOutputHandler)
 	output, err := runner.Run(ctx)
 
 	if postProcessOutput == valid.PostProcessRunOutputStripRefreshing {

server/core/terraform/terraform_client.go

@@ -466,7 +466,7 @@ func (c *DefaultClient) RunCommandAsync(ctx command.ProjectContext, path string,
 		envVars = append(envVars, fmt.Sprintf("%s=%s", key, val))
 	}
 
-	runner := models.NewShellCommandRunner(cmd, envVars, path, true, c.projectCmdOutputHandler)
+	runner := models.NewShellCommandRunner("sh", cmd, envVars, path, true, c.projectCmdOutputHandler)
 	inCh, outCh := runner.RunCommandAsync(ctx)
 	return inCh, outCh
 }