feat(policies): Add granular policy_sets

runatlantis.io/docs/custom-workflows.md

@@ -456,6 +456,8 @@ Or a custom command
   * `SHOWFILE` - Absolute path to the location where Atlantis expects the plan in json format to
   either be generated (by show) or already exist (if running policy checks). Can be used to
   override the built-in `plan`/`apply` commands, ex. `run: terraform show -json $PLANFILE > $SHOWFILE`.
+  * `POLICYCHECKFILE` - Absolute path to the location of policy check output if Atlantis runs policy checks.
+  See <LINK> for information of data structure.
   * `BASE_REPO_NAME` - Name of the repository that the pull request will be merged into, ex. `atlantis`.
   * `BASE_REPO_OWNER` - Owner of the repository that the pull request will be merged into, ex. `runatlantis`.
   * `HEAD_REPO_NAME` - Name of the repository that is getting merged into the base repository, ex. `atlantis`.

runatlantis.io/docs/policy-checking.md

@@ -158,3 +158,21 @@ workflows:
 ### Quiet policy checks
 
 By default, Atlantis will add a comment to all pull requests with the policy check result - both successes and failures. Version 0.21.0 added the [`--quiet-policy-checks`](server-configuration.html#quiet-policy-checks) option, which will instead only add comments when policy checks fail, significantly reducing the number of comments when most policy check results succeed.
+
+
+### Data for custom run steps
+
+When the policy check workflow runs, a file is created in the working directory which contains information about the status of each policy set tested. This data may be useful in custom run steps to generate metrics or notifications. The file contains JSON data in the following format:
+
+```json
+[
+  {
+    "PolicySetName":  "policy1",
+    "ConftestOutput": "",
+    "Passed":         false,
+    "ReqApprovals":   1,
+    "CurApprovals":   0
+  }
+]
+
+```

server/controllers/events/events_controller_e2e_test.go

@@ -1201,6 +1201,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 	Ok(t, err)
 
 	projectCommandRunner := &events.DefaultProjectCommandRunner{
+		VcsClient:        e2eVCSClient,
 		Locker:           projectLocker,
 		LockURLGenerator: &mockLockURLGenerator{},
 		InitStepRunner: &runtime.InitStepRunner{

server/controllers/events/testdata/test-repos/policy-checks-apply-reqs/exp-output-auto-policy-check.txt

@@ -1,15 +1,29 @@
 Ran Policy Check for dir: `.` workspace: `default`
 
-**Policy Check Error**
-```
-exit status 1
-Checking plan against the following policies: 
-  test_policy
+**Policy Check Failed**: Some policy sets did not pass.
+#### Policy Set: `test_policy`
+```diff
 FAIL - <redacted plan file> - main - WARNING: Null Resource creation is prohibited.
 
 1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions
 
 ```
-* :heavy_check_mark: To **approve** failing policies an authorized approver can comment:
+
+
+#### Policy Approval Status:
+```
+policy set: test_policy: requires: 1 approval(s), have: 0.
+```
+* :heavy_check_mark: To **approve** this project, comment:
+    * `atlantis approve_policies -d .`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan -d .`
+
+---
+* :heavy_check_mark: To **approve** all unapplied plans from this pull request, comment:
     * `atlantis approve_policies`
-* :repeat: Or, address the policy failure by modifying the codebase and re-planning.
+* :put_litter_in_its_place: To delete all plans and locks for the PR, comment:
+    * `atlantis unlock`
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan`

server/controllers/events/testdata/test-repos/policy-checks-diff-owner/exp-output-approve-policies.txt

@@ -1,4 +1,29 @@
+Ran Approve Policies for 1 projects:
+
+1. dir: `.` workspace: `default`
+
+### 1. dir: `.` workspace: `default`
 **Approve Policies Error**
 ```
-contact policy owners to approve failing policies
+1 error occurred:
+	* policy set: test_policy user runatlantis is not a policy owner - please contact policy owners to approve failing policies
+
+
 ```
+#### Policy Approval Status:
+```
+policy set: test_policy: requires: 1 approval(s), have: 0.
+```
+* :heavy_check_mark: To **approve** this project, comment:
+    * `atlantis approve_policies -d .`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan -d .`
+
+---
+* :heavy_check_mark: To **approve** all unapplied plans from this pull request, comment:
+    * `atlantis approve_policies`
+* :put_litter_in_its_place: To delete all plans and locks for the PR, comment:
+    * `atlantis unlock`
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan`

server/controllers/events/testdata/test-repos/policy-checks-diff-owner/exp-output-auto-policy-check.txt

@@ -1,15 +1,29 @@
 Ran Policy Check for dir: `.` workspace: `default`
 
-**Policy Check Error**
-```
-exit status 1
-Checking plan against the following policies: 
-  test_policy
+**Policy Check Failed**: Some policy sets did not pass.
+#### Policy Set: `test_policy`
+```diff
 FAIL - <redacted plan file> - main - WARNING: Null Resource creation is prohibited.
 
 1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions
 
 ```
-* :heavy_check_mark: To **approve** failing policies an authorized approver can comment:
+
+
+#### Policy Approval Status:
+```
+policy set: test_policy: requires: 1 approval(s), have: 0.
+```
+* :heavy_check_mark: To **approve** this project, comment:
+    * `atlantis approve_policies -d .`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan -d .`
+
+---
+* :heavy_check_mark: To **approve** all unapplied plans from this pull request, comment:
     * `atlantis approve_policies`
-* :repeat: Or, address the policy failure by modifying the codebase and re-planning.
+* :put_litter_in_its_place: To delete all plans and locks for the PR, comment:
+    * `atlantis unlock`
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan`

server/controllers/events/testdata/test-repos/policy-checks-extra-args/exp-output-auto-policy-check.txt

@@ -1,15 +1,29 @@
 Ran Policy Check for dir: `.` workspace: `default`
 
-**Policy Check Error**
-```
-exit status 1
-Checking plan against the following policies: 
-  test_policy
+**Policy Check Failed**: Some policy sets did not pass.
+#### Policy Set: `test_policy`
+```diff
 FAIL - <redacted plan file> - null_resource_policy - WARNING: Null Resource creation is prohibited.
 
 1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions
 
 ```
-* :heavy_check_mark: To **approve** failing policies an authorized approver can comment:
+
+
+#### Policy Approval Status:
+```
+policy set: test_policy: requires: 1 approval(s), have: 0.
+```
+* :heavy_check_mark: To **approve** this project, comment:
+    * `atlantis approve_policies -d .`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan -d .`
+
+---
+* :heavy_check_mark: To **approve** all unapplied plans from this pull request, comment:
     * `atlantis approve_policies`
-* :repeat: Or, address the policy failure by modifying the codebase and re-planning.
+* :put_litter_in_its_place: To delete all plans and locks for the PR, comment:
+    * `atlantis unlock`
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan`

server/controllers/events/testdata/test-repos/policy-checks-multi-projects/exp-output-approve-policies.txt

@@ -3,3 +3,4 @@ Approved Policies for 1 projects:
 1. dir: `.` workspace: `default`
 
 
+i

server/controllers/events/testdata/test-repos/policy-checks-multi-projects/exp-output-auto-policy-check.txt

@@ -4,13 +4,14 @@ Ran Policy Check for 2 projects:
 1. dir: `dir2` workspace: `default`
 
 ### 1. dir: `dir1` workspace: `default`
+#### Policy Set: `test_policy`
 ```diff
-Checking plan against the following policies: 
-  test_policy
 
 1 test, 1 passed, 0 warnings, 0 failures, 0 exceptions
+
 ```
 
+
 * :arrow_forward: To **apply** this plan, comment:
     * `atlantis apply -d dir1`
 * :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
@@ -19,22 +20,30 @@ Checking plan against the following policies:
 
 ---
 ### 2. dir: `dir2` workspace: `default`
-**Policy Check Error**
-```
-exit status 1
-Checking plan against the following policies: 
-  test_policy
+**Policy Check Failed**: Some policy sets did not pass.
+#### Policy Set: `test_policy`
+```diff
 FAIL - <redacted plan file> - main - WARNING: Forbidden Resource creation is prohibited.
 
 1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions
 
 ```
-* :heavy_check_mark: To **approve** failing policies an authorized approver can comment:
-    * `atlantis approve_policies`
-* :repeat: Or, address the policy failure by modifying the codebase and re-planning.
+
+
+#### Policy Approval Status:
+```
+policy set: test_policy: requires: 1 approval(s), have: 0.
+```
+* :heavy_check_mark: To **approve** this project, comment:
+    * `atlantis approve_policies -d dir2`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan -d dir2`
 
 ---
-* :fast_forward: To **apply** all unapplied plans from this pull request, comment:
-    * `atlantis apply`
+* :heavy_check_mark: To **approve** all unapplied plans from this pull request, comment:
+    * `atlantis approve_policies`
 * :put_litter_in_its_place: To delete all plans and locks for the PR, comment:
-    * `atlantis unlock`
+    * `atlantis unlock`
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan`

server/controllers/events/testdata/test-repos/policy-checks/exp-output-auto-policy-check.txt

@@ -1,15 +1,29 @@
 Ran Policy Check for dir: `.` workspace: `default`
 
-**Policy Check Error**
-```
-exit status 1
-Checking plan against the following policies: 
-  test_policy
+**Policy Check Failed**: Some policy sets did not pass.
+#### Policy Set: `test_policy`
+```diff
 FAIL - <redacted plan file> - main - WARNING: Null Resource creation is prohibited.
 
 1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions
 
 ```
-* :heavy_check_mark: To **approve** failing policies an authorized approver can comment:
+
+
+#### Policy Approval Status:
+```
+policy set: test_policy: requires: 1 approval(s), have: 0.
+```
+* :heavy_check_mark: To **approve** this project, comment:
+    * `atlantis approve_policies -d .`
+* :put_litter_in_its_place: To **delete** this plan click [here](lock-url)
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan -d .`
+
+---
+* :heavy_check_mark: To **approve** all unapplied plans from this pull request, comment:
     * `atlantis approve_policies`
-* :repeat: Or, address the policy failure by modifying the codebase and re-planning.
+* :put_litter_in_its_place: To delete all plans and locks for the PR, comment:
+    * `atlantis unlock`
+* :repeat: To re-run policies **plan** this project again by commenting:
+    * `atlantis plan`

server/core/config/parser_validator_test.go

@@ -1467,11 +1467,13 @@ policies:
 				},
 				PolicySets: valid.PolicySets{
 					Version: conftestVersion,
+				    ReviewCount: 1,
 					PolicySets: []valid.PolicySet{
 						{
 							Name:   "good-policy",
 							Path:   "rel/path/to/policy",
 							Source: valid.LocalPolicySet,
+				            ReviewCount: 1,
 						},
 					},
 				},
@@ -1802,11 +1804,13 @@ func TestParserValidator_ParseGlobalCfgJSON(t *testing.T) {
 				},
 				PolicySets: valid.PolicySets{
 					Version: conftestVersion,
+				    ReviewCount: 1,
 					PolicySets: []valid.PolicySet{
 						{
 							Name:   "good-policy",
 							Path:   "rel/path/to/policy",
 							Source: valid.LocalPolicySet,
+				            ReviewCount: 1,
 						},
 					},
 				},

server/core/config/raw/policies.go

@@ -2,15 +2,16 @@ package raw
 
 import (
 	validation "github.com/go-ozzo/ozzo-validation"
-	"github.com/hashicorp/go-version"
+	version "github.com/hashicorp/go-version"
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 )
 
 // PolicySets is the raw schema for repo-level atlantis.yaml config.
 type PolicySets struct {
-	Version    *string      `yaml:"conftest_version,omitempty" json:"conftest_version,omitempty"`
-	Owners     PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
-	PolicySets []PolicySet  `yaml:"policy_sets" json:"policy_sets"`
+	Version     *string      `yaml:"conftest_version,omitempty" json:"conftest_version,omitempty"`
+	Owners      PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	PolicySets  []PolicySet  `yaml:"policy_sets" json:"policy_sets"`
+	ReviewCount int          `yaml:"review_count,omitempty" json:"review_count,omitempty"`
 }
 
 func (p PolicySets) Validate() error {
@@ -27,10 +28,20 @@ func (p PolicySets) ToValid() valid.PolicySets {
 		policySets.Version, _ = version.NewVersion(*p.Version)
 	}
 
+	// Default number of required reviews for all policy sets should be 1.
+	policySets.ReviewCount = p.ReviewCount
+	if policySets.ReviewCount == 0 {
+		policySets.ReviewCount = 1
+	}
+
 	policySets.Owners = p.Owners.ToValid()
 
 	validPolicySets := make([]valid.PolicySet, 0)
 	for _, rawPolicySet := range p.PolicySets {
+		// Default to top-level review count if not specified.
+		if rawPolicySet.ReviewCount == 0 {
+			rawPolicySet.ReviewCount = policySets.ReviewCount
+		}
 		validPolicySets = append(validPolicySets, rawPolicySet.ToValid())
 	}
 	policySets.PolicySets = validPolicySets
@@ -57,16 +68,18 @@ func (o PolicyOwners) ToValid() valid.PolicyOwners {
 }
 
 type PolicySet struct {
-	Path   string       `yaml:"path" json:"path"`
-	Source string       `yaml:"source" json:"source"`
-	Name   string       `yaml:"name" json:"name"`
-	Owners PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	Path        string       `yaml:"path" json:"path"`
+	Source      string       `yaml:"source" json:"source"`
+	Name        string       `yaml:"name" json:"name"`
+	Owners      PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	ReviewCount int          `yaml:"review_count,omitempty" json:"review_count,omitempty"`
 }
 
 func (p PolicySet) Validate() error {
 	return validation.ValidateStruct(&p,
 		validation.Field(&p.Name, validation.Required.Error("is required")),
 		validation.Field(&p.Owners),
+		validation.Field(&p.ReviewCount),
 		validation.Field(&p.Path, validation.Required.Error("is required")),
 		validation.Field(&p.Source, validation.In(valid.LocalPolicySet, valid.GithubPolicySet).Error("only 'local' and 'github' source types are supported")),
 	)
@@ -78,6 +91,7 @@ func (p PolicySet) ToValid() valid.PolicySet {
 	policySet.Name = p.Name
 	policySet.Path = p.Path
 	policySet.Source = p.Source
+	policySet.ReviewCount = p.ReviewCount
 	policySet.Owners = p.Owners.ToValid()
 
 	return policySet