feat: Add option allowing usage of custom policy check tools (#3765)

* Adding new flag everywhere relevant, implementing policy result workaround

* Fixing unit test str matching, adding custom policy conditional to step_runner

* Adding documentation steps for custom policy tools

* Refactoring ConftestOutput attribute to PolicyOutput

runatlantis.io/docs/custom-policy-checks.md

@@ -0,0 +1,45 @@
+# Custom Policy Checks
+If you want to run custom policy tools or scripts instead of the built-in Conftest integration, you can do so by setting the `custom_policy_check` option and running it in a custom workflow.  Note: custom policy tool output is simply parsed for "fail" substrings to determine if the policy set passed. 
+
+This option can be configured either at the server-level in a [repos.yaml config file](server-configuration.md) or at the repo-level in an [atlantis.yaml file.](repo-level-atlantis-yaml.md). 
+
+## Server-side config example
+Set the `policy_check` and `custom_policy_check` options to true, and run the custom tool in the policy check steps as seen below.  No
+
+```yaml
+repos:
+  - id: /.*/
+    branch: /^main$/
+    apply_requirements: [mergeable, undiverged, approved]
+    policy_check: true
+    custom_policy_check: true
+    workflow: custom
+workflows:
+  custom:
+    policy_check:
+      steps:
+        - show
+        - run: cnspec scan terraform plan $SHOWFILE --policy-bundle example-cnspec-policies.mql.yaml 
+policies:
+  owners:
+    users:
+      - example_ghuser
+  policy_sets:
+    - name: example-set
+      path: example-cnspec-policies.mql.yaml 
+      source: local
+```
+
+
+## Repo-level atlantis.yaml example
+First, you will need to ensure `custom_policy_check` is within the `allowed_overrides` field of the server-side config.  Next, just set the custom option to true on the specific project you want as shown in the example `atlantis.yaml` below:
+
+```yaml
+version: 3
+projects:
+  - name: example
+    dir: ./example
+    custom_policy_check: true
+    autoplan:
+      when_modified: ["*.tf"]
+```

runatlantis.io/docs/policy-checking.md

@@ -187,7 +187,7 @@ When the policy check workflow runs, a file is created in the working directory
 [
   {
     "PolicySetName":  "policy1",
-    "ConftestOutput": "",
+    "PolicyOutput": "",
     "Passed":         false,
     "ReqApprovals":   1,
     "CurApprovals":   0

runatlantis.io/docs/repo-level-atlantis-yaml.md

@@ -59,6 +59,7 @@ projects:
   terraform_version: v0.11.0
   delete_source_branch_on_merge: true
   repo_locking: true
+  custom_policy_check: false
   autoplan:
     when_modified: ["*.tf", "../modules/**/*.tf", ".terraform.lock.hcl"]
     enabled: true
@@ -311,6 +312,7 @@ workspace: myworkspace
 execution_order_group: 0
 delete_source_branch_on_merge: false
 repo_locking: true
+custom_policy_check: false
 autoplan:
 terraform_version: 0.11.0
 plan_requirements: ["approved"]
@@ -327,7 +329,9 @@ workflow: myworkflow
 | workspace                                | string                | `"default"` | no       | The [Terraform workspace](https://developer.hashicorp.com/terraform/language/state/workspaces) for this project. Atlantis will switch to this workplace when planning/applying and will create it if it doesn't exist.                    |
 | execution_order_group                    | int                   | `0`         | no       | Index of execution order group. Projects will be sort by this field before planning/applying.                                                                                                                                             |
 | delete_source_branch_on_merge            | bool                  | `false`     | no       | Automatically deletes the source branch on merge.                                                                                                                                                                                         |
-| repo_locking                             | bool                  | `true`      | no       | Get a repository lock in this project when plan.                                                                                                                                                                                          |
+| repo_locking                             | bool                  | `true`      | no       | Get a repository lock in this project when plan.                                                                         
+
+| custom_policy_check                             | bool                  | `false`      | no       | Enable using policy check tools other than Conftest                                                                                                                                                           |
 | autoplan                                 | [Autoplan](#autoplan) | none        | no       | A custom autoplan configuration. If not specified, will use the autoplan config. See [Autoplanning](autoplanning.html).                                                                                                                   |
 | terraform_version                        | string                | none        | no       | A specific Terraform version to use when running commands for this project. Must be [Semver compatible](https://semver.org/), ex. `v0.11.0`, `0.12.0-beta1`.                                                                              |
 | plan_requirements<br />*(restricted)*   | array[string]         | none        | no       | Requirements that must be satisfied before `atlantis plan` can be run. Currently the only supported requirements are `approved`, `mergeable`, and `undiverged`. See [Command Requirements](command-requirements.html) for more details.  |

runatlantis.io/docs/server-side-repo-config.md

@@ -54,7 +54,7 @@ repos:
 
   # allowed_overrides specifies which keys can be overridden by this repo in
   # its atlantis.yaml file.
-  allowed_overrides: [apply_requirements, workflow, delete_source_branch_on_merge, repo_locking]
+  allowed_overrides: [apply_requirements, workflow, delete_source_branch_on_merge, repo_locking, custom_policy_check]
 
   # allowed_workflows specifies which workflows the repos that match 
   # are allowed to select.
@@ -73,6 +73,10 @@ repos:
   # If true (default), atlantis try to get a lock.
   repo_locking: true
 
+  # custom_policy_check defines whether policy checking tools besides Conftest are enabled in checks
+  # If false (default), only Conftest JSON output is allowed
+  custom_policy_check: false
+
   # pre_workflow_hooks defines arbitrary list of scripts to execute before workflow execution.
   pre_workflow_hooks: 
     - run: my-pre-workflow-hook-command arg1
@@ -340,6 +344,9 @@ unless you've created your own server-side workflow with that key (overriding it
 See [Custom Workflows](custom-workflows.html) for more details on writing
 custom workflows.
 
+### Allow Using Custom Policy Tools
+Conftest is the standard policy check application integrated with Atlantis, but custom tools can still be run in custom workflows when the `custom_policy_check` option is set.  See the [Custom Policy Checks page](custom-policy-checks.md) for detailed examples.
+
 ### Allow Repos To Define Their Own Workflows
 If you want repos to be able to define their own workflows you need to
 allow them to override the `workflow` key and set `allow_custom_workflows` to `true`.
@@ -482,12 +489,13 @@ If you set a workflow with the key `default`, it will override this.
 | plan_requirements            | []string | none    | no       | Requirements that must be satisfied before `atlantis plan` can be run. Currently the only supported requirements are `approved`, `mergeable`, and `undiverged`. See [Command Requirements](command-requirements.html) for more details.                                                                  |                                                                                           |
 | apply_requirements            | []string | none    | no       | Requirements that must be satisfied before `atlantis apply` can be run. Currently the only supported requirements are `approved`, `mergeable`, and `undiverged`. See [Command Requirements](command-requirements.html) for more details.                                                                  |
 | import_requirements           | []string | none    | no       | Requirements that must be satisfied before `atlantis import` can be run. Currently the only supported requirements are `approved`, `mergeable`, and `undiverged`. See [Command Requirements](command-requirements.html) for more details.                                                                 |
-| allowed_overrides             | []string | none    | no       | A list of restricted keys that `atlantis.yaml` files can override. The only supported keys are `apply_requirements`, `workflow`, `delete_source_branch_on_merge` and `repo_locking`                                                                                                                       |
+| allowed_overrides             | []string | none    | no       | A list of restricted keys that `atlantis.yaml` files can override. The only supported keys are `apply_requirements`, `workflow`, `delete_source_branch_on_merge`,`repo_locking`, and `custom_policy_check`                                                                                                                          |
 | allowed_workflows             | []string | none    | no       | A list of workflows that `atlantis.yaml` files can select from.                                                                                                                                                                                                                                           |
 | allow_custom_workflows        | bool     | false   | no       | Whether or not to allow [Custom Workflows](custom-workflows.html).                                                                                                                                                                                                                                        |
 | delete_source_branch_on_merge | bool     | false   | no       | Whether or not to delete the source branch on merge.                                                                                                                                                                                                                                                      |
 | repo_locking                  | bool     | false   | no       | Whether or not to get a lock.                                                                                                                                                                                                                                                                             |
 | policy_check                  | bool     | false   | no       | Whether or not to run policy checks on this repository.                                                                                                                                                                                                                                                   |
+| custom_policy_check                  | bool     | false   | no       | Whether or not to enable custom policy check tools outside of Conftest on this repository.                                                                                                                                                                                                       |
 
 
 :::tip Notes

server/core/config/parser_validator_test.go

@@ -1309,7 +1309,7 @@ func TestParseGlobalCfg(t *testing.T) {
 			input: `repos:
 - id: /.*/
   allowed_overrides: [invalid]`,
-			expErr: "repos: (0: (allowed_overrides: \"invalid\" is not a valid override, only \"plan_requirements\", \"apply_requirements\", \"import_requirements\", \"workflow\", \"delete_source_branch_on_merge\", \"repo_locking\" and \"policy_check\" are supported.).).",
+			expErr: "repos: (0: (allowed_overrides: \"invalid\" is not a valid override, only \"plan_requirements\", \"apply_requirements\", \"import_requirements\", \"workflow\", \"delete_source_branch_on_merge\", \"repo_locking\", \"policy_check\", and \"custom_policy_check\" are supported.).).",
 		},
 		"invalid plan_requirement": {
 			input: `repos:
@@ -1573,6 +1573,7 @@ workflows:
 						DeleteSourceBranchOnMerge: Bool(false),
 						RepoLocking:               Bool(true),
 						PolicyCheck:               Bool(false),
+						CustomPolicyCheck:         Bool(false),
 					},
 				},
 				Workflows: map[string]valid.Workflow{

server/core/config/raw/global_cfg.go

@@ -35,6 +35,7 @@ type Repo struct {
 	DeleteSourceBranchOnMerge *bool          `yaml:"delete_source_branch_on_merge,omitempty" json:"delete_source_branch_on_merge,omitempty"`
 	RepoLocking               *bool          `yaml:"repo_locking,omitempty" json:"repo_locking,omitempty"`
 	PolicyCheck               *bool          `yaml:"policy_check,omitempty" json:"policy_check,omitempty"`
+	CustomPolicyCheck         *bool          `yaml:"custom_policy_check,omitempty" json:"custom_policy_check,omitempty"`
 }
 
 func (g GlobalCfg) Validate() error {
@@ -192,8 +193,8 @@ func (r Repo) Validate() error {
 	overridesValid := func(value interface{}) error {
 		overrides := value.([]string)
 		for _, o := range overrides {
-			if o != valid.PlanRequirementsKey && o != valid.ApplyRequirementsKey && o != valid.ImportRequirementsKey && o != valid.WorkflowKey && o != valid.DeleteSourceBranchOnMergeKey && o != valid.RepoLockingKey && o != valid.PolicyCheckKey {
-				return fmt.Errorf("%q is not a valid override, only %q, %q, %q, %q, %q, %q and %q are supported", o, valid.PlanRequirementsKey, valid.ApplyRequirementsKey, valid.ImportRequirementsKey, valid.WorkflowKey, valid.DeleteSourceBranchOnMergeKey, valid.RepoLockingKey, valid.PolicyCheckKey)
+			if o != valid.PlanRequirementsKey && o != valid.ApplyRequirementsKey && o != valid.ImportRequirementsKey && o != valid.WorkflowKey && o != valid.DeleteSourceBranchOnMergeKey && o != valid.RepoLockingKey && o != valid.PolicyCheckKey && o != valid.CustomPolicyCheckKey {
+				return fmt.Errorf("%q is not a valid override, only %q, %q, %q, %q, %q, %q, %q, and %q are supported", o, valid.PlanRequirementsKey, valid.ApplyRequirementsKey, valid.ImportRequirementsKey, valid.WorkflowKey, valid.DeleteSourceBranchOnMergeKey, valid.RepoLockingKey, valid.PolicyCheckKey, valid.CustomPolicyCheckKey)
 			}
 		}
 		return nil
@@ -331,5 +332,6 @@ OuterGlobalImportReqs:
 		DeleteSourceBranchOnMerge: r.DeleteSourceBranchOnMerge,
 		RepoLocking:               r.RepoLocking,
 		PolicyCheck:               r.PolicyCheck,
+		CustomPolicyCheck:         r.CustomPolicyCheck,
 	}
 }

server/core/config/raw/project.go

@@ -36,6 +36,7 @@ type Project struct {
 	RepoLocking               *bool     `yaml:"repo_locking,omitempty"`
 	ExecutionOrderGroup       *int      `yaml:"execution_order_group,omitempty"`
 	PolicyCheck               *bool     `yaml:"policy_check,omitempty"`
+	CustomPolicyCheck         *bool     `yaml:"custom_policy_check,omitempty"`
 }
 
 func (p Project) Validate() error {
@@ -146,6 +147,10 @@ func (p Project) ToValid() valid.Project {
 		v.PolicyCheck = p.PolicyCheck
 	}
 
+	if p.CustomPolicyCheck != nil {
+		v.CustomPolicyCheck = p.CustomPolicyCheck
+	}
+
 	return v
 }