Initial work.

server/core/config/raw/policies.go

@@ -2,15 +2,16 @@ package raw
 
 import (
 	validation "github.com/go-ozzo/ozzo-validation"
-	"github.com/hashicorp/go-version"
+	version "github.com/hashicorp/go-version"
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 )
 
 // PolicySets is the raw schema for repo-level atlantis.yaml config.
 type PolicySets struct {
-	Version    *string      `yaml:"conftest_version,omitempty" json:"conftest_version,omitempty"`
-	Owners     PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
-	PolicySets []PolicySet  `yaml:"policy_sets" json:"policy_sets"`
+	Version     *string      `yaml:"conftest_version,omitempty" json:"conftest_version,omitempty"`
+	Owners      PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	PolicySets  []PolicySet  `yaml:"policy_sets" json:"policy_sets"`
+	ReviewCount int          `yaml:"review_count,omitempty" json:"review_count,omitempty"`
 }
 
 func (p PolicySets) Validate() error {
@@ -27,10 +28,20 @@ func (p PolicySets) ToValid() valid.PolicySets {
 		policySets.Version, _ = version.NewVersion(*p.Version)
 	}
 
+	// Default number of required reviews for all policy sets should be 1.
+	policySets.ReviewCount = p.ReviewCount
+	if policySets.ReviewCount == 0 {
+		policySets.ReviewCount = 1
+	}
+
 	policySets.Owners = p.Owners.ToValid()
 
 	validPolicySets := make([]valid.PolicySet, 0)
 	for _, rawPolicySet := range p.PolicySets {
+		// Default to top-level review count if not specified.
+		if rawPolicySet.ReviewCount == 0 {
+			rawPolicySet.ReviewCount = policySets.ReviewCount
+		}
 		validPolicySets = append(validPolicySets, rawPolicySet.ToValid())
 	}
 	policySets.PolicySets = validPolicySets
@@ -57,16 +68,18 @@ func (o PolicyOwners) ToValid() valid.PolicyOwners {
 }
 
 type PolicySet struct {
-	Path   string       `yaml:"path" json:"path"`
-	Source string       `yaml:"source" json:"source"`
-	Name   string       `yaml:"name" json:"name"`
-	Owners PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	Path        string       `yaml:"path" json:"path"`
+	Source      string       `yaml:"source" json:"source"`
+	Name        string       `yaml:"name" json:"name"`
+	Owners      PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	ReviewCount int          `yaml:"review_count,omitempty" json:"review_count,omitempty"`
 }
 
 func (p PolicySet) Validate() error {
 	return validation.ValidateStruct(&p,
 		validation.Field(&p.Name, validation.Required.Error("is required")),
 		validation.Field(&p.Owners),
+		validation.Field(&p.ReviewCount),
 		validation.Field(&p.Path, validation.Required.Error("is required")),
 		validation.Field(&p.Source, validation.In(valid.LocalPolicySet, valid.GithubPolicySet).Error("only 'local' and 'github' source types are supported")),
 	)
@@ -78,6 +91,7 @@ func (p PolicySet) ToValid() valid.PolicySet {
 	policySet.Name = p.Name
 	policySet.Path = p.Path
 	policySet.Source = p.Source
+	policySet.ReviewCount = p.ReviewCount
 	policySet.Owners = p.Owners.ToValid()
 
 	return policySet

server/core/config/valid/policies.go

@@ -3,7 +3,7 @@ package valid
 import (
 	"strings"
 
-	"github.com/hashicorp/go-version"
+	version "github.com/hashicorp/go-version"
 )
 
 const (
@@ -15,9 +15,10 @@ const (
 // PolicySet objects. PolicySets struct is used by PolicyCheck workflow to build
 // context to enforce policies.
 type PolicySets struct {
-	Version    *version.Version
-	Owners     PolicyOwners
-	PolicySets []PolicySet
+	Version     *version.Version
+	Owners      PolicyOwners
+	ReviewCount int
+	PolicySets  []PolicySet
 }
 
 type PolicyOwners struct {
@@ -26,10 +27,11 @@ type PolicyOwners struct {
 }
 
 type PolicySet struct {
-	Source string
-	Path   string
-	Name   string
-	Owners PolicyOwners
+	Source      string
+	Path        string
+	Name        string
+	ReviewCount int
+	Owners      PolicyOwners
 }
 
 func (p *PolicySets) HasPolicies() bool {

server/core/db/boltdb.go

@@ -357,6 +357,7 @@ func (b *BoltDB) UpdatePullWithResults(pull models.PullRequest, newResults []com
 						res.ProjectName == proj.ProjectName {
 
 						proj.Status = res.PlanStatus()
+						proj.PolicyStatus = res.PolicyCheckApprovals
 						updatedExisting = true
 						break
 					}
@@ -483,9 +484,10 @@ func (b *BoltDB) writePullToBucket(bucket *bolt.Bucket, key []byte, pull models.
 
 func (b *BoltDB) projectResultToProject(p command.ProjectResult) models.ProjectStatus {
 	return models.ProjectStatus{
-		Workspace:   p.Workspace,
-		RepoRelDir:  p.RepoRelDir,
-		ProjectName: p.ProjectName,
-		Status:      p.PlanStatus(),
+		Workspace:    p.Workspace,
+		RepoRelDir:   p.RepoRelDir,
+		ProjectName:  p.ProjectName,
+		PolicyStatus: p.PolicyCheckApprovals,
+		Status:       p.PlanStatus(),
 	}
 }

server/core/runtime/policy/conftest_client.go

@@ -17,6 +17,10 @@ import (
 	"github.com/runatlantis/atlantis/server/logging"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
+	"github.com/hashicorp/go-multierror"
+	"github.com/runatlantis/atlantis/server/events/models"
+	"encoding/json"
+	"regexp"
 )
 
 const (
@@ -163,46 +167,64 @@ func NewConfTestExecutorWorkflow(log logging.SimpleLogging, versionRootDir strin
 }
 
 func (c *ConfTestExecutorWorkflow) Run(ctx command.ProjectContext, executablePath string, envs map[string]string, workdir string, extraArgs []string) (string, error) {
-	policyArgs := []Arg{}
-	policySetNames := []string{}
 	ctx.Log.Debug("policy sets, %s ", ctx.PolicySets)
+
+	inputFile := filepath.Join(workdir, ctx.GetShowResultFileName())
+	var policySetResults []models.PolicySetResult
+	var combinedErr error
+
+
 	for _, policySet := range ctx.PolicySets.PolicySets {
-		path, err := c.SourceResolver.Resolve(policySet)
+		path, resolveErr := c.SourceResolver.Resolve(policySet)
 
 		// Let's not fail the whole step because of a single failure. Log and fail silently
-		if err != nil {
-			ctx.Log.Err("Error resolving policyset %s. err: %s", policySet.Name, err.Error())
+		if resolveErr != nil {
+			ctx.Log.Err("Error resolving policyset %s. err: %s", policySet.Name, resolveErr.Error())
 			continue
 		}
 
-		policyArg := NewPolicyArg(path)
-		policyArgs = append(policyArgs, policyArg)
-
-		policySetNames = append(policySetNames, policySet.Name)
-	}
-
-	inputFile := filepath.Join(workdir, ctx.GetShowResultFileName())
+		args := ConftestTestCommandArgs{
+		   	PolicyArgs: []Arg{NewPolicyArg(path)},
+		   	ExtraArgs:  extraArgs,
+		   	InputFile:  inputFile,
+		   	Command:    executablePath,
+		}
 
-	args := ConftestTestCommandArgs{
-		PolicyArgs: policyArgs,
-		ExtraArgs:  extraArgs,
-		InputFile:  inputFile,
-		Command:    executablePath,
+	    serializedArgs, _ := args.build()
+	    cmdOutput, cmdErr := c.Exec.CombinedOutput(serializedArgs, envs, workdir)
+
+	    passed := true
+	    if cmdErr != nil {
+	        // Since we're running conftest for each policyset, individual command errors should be concatenated.
+	        if isValidConftestOutput(cmdOutput) {
+	            combinedErr = multierror.Append(combinedErr, errors.New(fmt.Sprintf("policy_set: %s: conftest: %s", policySet.Name, "Some policies failed.")))
+	        } else {
+	            combinedErr = multierror.Append(combinedErr, errors.New(fmt.Sprintf("policy_set: %s: conftest: %s", policySet.Name, cmdOutput)))
+	        }
+	        passed = false
+	    }
+
+	    policySetResults = append(policySetResults, models.PolicySetResult{
+	        PolicySetName:   policySet.Name,
+	        PolicySetOutput: cmdOutput,
+	        Passed:          passed,
+	    })
 	}
 
-	serializedArgs, err := args.build()
-
-	if err != nil {
-		ctx.Log.Warn("No policies have been configured")
+	if policySetResults == nil {
+		ctx.Log.Warn("No policies have been configured.")
 		return "", nil
 		// TODO: enable when we can pass policies in otherwise e2e tests with policy checks fail
 		// return "", errors.Wrap(err, "building args")
 	}
 
-	initialOutput := fmt.Sprintf("Checking plan against the following policies: \n  %s\n", strings.Join(policySetNames, "\n  "))
-	cmdOutput, err := c.Exec.CombinedOutput(serializedArgs, envs, workdir)
+	marshaledStatus, err := json.Marshal(policySetResults)
+	if err != nil {
+	    return "", errors.New(fmt.Sprintf("Cannot marshal data into []PolicySetResult. Error: %w Data: %w", err, policySetResults))
+	}
+	output := string(marshaledStatus)
 
-	return c.sanitizeOutput(inputFile, initialOutput+cmdOutput), err
+	return c.sanitizeOutput(inputFile, output), combinedErr
 
 }
 
@@ -255,3 +277,13 @@ func getDefaultVersion() (*version.Version, error) {
 	}
 	return wrappedVersion, nil
 }
+
+// Checks if output from conftest is a valid output.
+func isValidConftestOutput(output string) bool {
+
+	r := regexp.MustCompile(`^(WARN|FAIL|\[)`)
+	if match := r.FindString(output); match != "" {
+		return true
+	}
+	return false
+}

server/events/command/project_result.go

@@ -6,19 +6,20 @@ import (
 
 // ProjectResult is the result of executing a plan/policy_check/apply for a specific project.
 type ProjectResult struct {
-	Command            Name
-	SubCommand         string
-	RepoRelDir         string
-	Workspace          string
-	Error              error
-	Failure            string
-	PlanSuccess        *models.PlanSuccess
-	PolicyCheckSuccess *models.PolicyCheckSuccess
-	ApplySuccess       string
-	VersionSuccess     string
-	ImportSuccess      *models.ImportSuccess
-	StateRmSuccess     *models.StateRmSuccess
-	ProjectName        string
+	Command              Name
+	SubCommand           string
+	RepoRelDir           string
+	Workspace            string
+	Error                error
+	Failure              string
+	PlanSuccess          *models.PlanSuccess
+	PolicyCheckSuccess   *models.PolicyCheckSuccess
+	PolicyCheckApprovals []models.PolicySetApproval
+	ApplySuccess         string
+	VersionSuccess       string
+	ImportSuccess        *models.ImportSuccess
+	StateRmSuccess       *models.StateRmSuccess
+	ProjectName          string
 }
 
 // CommitStatus returns the vcs commit status of this project result.

server/events/markdown_renderer.go

@@ -78,7 +78,14 @@ type errData struct {
 	commonData
 }
 
-// failureData is data about a failure response.
+// policyFailureData is data about a failing
+type policyCheckFailureData struct {
+    Failure string
+    commonData
+    models.PolicyCheckSuccess
+}
+
+// failureData is data about a generic failure response.
 type failureData struct {
 	Failure string
 	commonData
@@ -189,7 +196,11 @@ func (m *MarkdownRenderer) renderProjectResults(results []command.ProjectResult,
 			}
 			resultData.Rendered = m.renderTemplateTrimSpace(tmpl, errData{result.Error.Error(), common})
 		} else if result.Failure != "" {
-			resultData.Rendered = m.renderTemplateTrimSpace(templates.Lookup("failure"), failureData{result.Failure, common})
+		    if common.Command == "PolicyCheck" {
+		        resultData.Rendered = m.renderTemplateTrimSpace(templates.Lookup("failure"), policyCheckFailureData{result.Failure, common, *result.PolicyCheckSuccess})
+		    } else {
+		        resultData.Rendered = m.renderTemplateTrimSpace(templates.Lookup("failure"), failureData{result.Failure, common})
+		    }
 		} else if result.PlanSuccess != nil {
 			result.PlanSuccess.TerraformOutput = strings.TrimSpace(result.PlanSuccess.TerraformOutput)
 			if m.shouldUseWrappedTmpl(vcsHost, result.PlanSuccess.TerraformOutput) {
@@ -199,12 +210,12 @@ func (m *MarkdownRenderer) renderProjectResults(results []command.ProjectResult,
 			}
 			numPlanSuccesses++
 		} else if result.PolicyCheckSuccess != nil {
-			result.PolicyCheckSuccess.PolicyCheckOutput = strings.TrimSpace(result.PolicyCheckSuccess.PolicyCheckOutput)
-			if m.shouldUseWrappedTmpl(vcsHost, result.PolicyCheckSuccess.PolicyCheckOutput) {
-				resultData.Rendered = m.renderTemplateTrimSpace(templates.Lookup("policyCheckSuccessWrapped"), policyCheckSuccessData{PolicyCheckSuccess: *result.PolicyCheckSuccess, PolicyCheckSummary: result.PolicyCheckSuccess.Summary()})
-			} else {
-				resultData.Rendered = m.renderTemplateTrimSpace(templates.Lookup("policyCheckSuccessUnwrapped"), policyCheckSuccessData{PolicyCheckSuccess: *result.PolicyCheckSuccess})
-			}
+//			result.PolicyCheckSuccess.PolicySetResults = strings.TrimSpace(string(result.PolicyCheckSuccess.PolicySetResults))
+//			if m.shouldUseWrappedTmpl(vcsHost, result.PolicyCheckSuccess.PolicySetResults) {
+//				resultData.Rendered = m.renderTemplateTrimSpace(templates.Lookup("policyCheckSuccessWrapped"), policyCheckSuccessData{PolicyCheckSuccess: *result.PolicyCheckSuccess, PolicyCheckSummary: result.PolicyCheckSuccess.Summary()})
+//			} else {
+			resultData.Rendered = m.renderTemplateTrimSpace(templates.Lookup("policyCheckSuccessUnwrapped"), policyCheckSuccessData{PolicyCheckSuccess: *result.PolicyCheckSuccess})
+//			}
 			numPolicyCheckSuccesses++
 		} else if result.ApplySuccess != "" {
 			output := strings.TrimSpace(result.ApplySuccess)

server/events/models/models.go

@@ -366,6 +366,12 @@ type PlanSuccess struct {
 	HasDiverged bool
 }
 
+type PolicySetResult struct {
+	PolicySetName   string
+	PolicySetOutput string
+	Passed          bool
+}
+
 // Summary regexes
 var (
 	reChangesOutside = regexp.MustCompile(`Note: Objects have changed outside of Terraform`)
@@ -409,7 +415,7 @@ func (p PlanSuccess) DiffMarkdownFormattedTerraformOutput() string {
 // PolicyCheckSuccess is the result of a successful policy check run.
 type PolicyCheckSuccess struct {
 	// PolicyCheckOutput is the output from policy check binary(conftest|opa)
-	PolicyCheckOutput string
+	PolicySetResults []PolicySetResult
 	// LockURL is the full URL to the lock held by this policy check.
 	LockURL string
 	// RePlanCmd is the command that users should run to re-plan this project.
@@ -441,11 +447,11 @@ type StateRmSuccess struct {
 // Summary extracts one line summary of policy check.
 func (p *PolicyCheckSuccess) Summary() string {
 	note := ""
-
-	r := regexp.MustCompile(`\d+ tests?, \d+ passed, \d+ warnings?, \d+ failures?, \d+ exceptions?(, \d skipped)?`)
-	if match := r.FindString(p.PolicyCheckOutput); match != "" {
-		return note + match
-	}
+	//
+	//r := regexp.MustCompile(`\d+ tests?, \d+ passed, \d+ warnings?, \d+ failures?, \d+ exceptions?(, \d skipped)?`)
+	//if match := r.FindString(p.PolicyCheckOutput); match != "" {
+	//	return note + match
+	//}
 	return note
 }
 
@@ -472,11 +478,19 @@ func (p PullStatus) StatusCount(status ProjectPlanStatus) int {
 	return c
 }
 
+// PolicySetApprovalStatus tracks the number of approvals a given PolicySet has received.
+type PolicySetApproval struct {
+	PolicySetName string
+	Approvals     int
+}
+
 // ProjectStatus is the status of a specific project.
 type ProjectStatus struct {
 	Workspace   string
 	RepoRelDir  string
 	ProjectName string
+	// PolicySetApprovals tracks the approval status of every PolicySet for a Project.
+	PolicyStatus []PolicySetApproval
 	// Status is the status of where this project is at in the planning cycle.
 	Status ProjectPlanStatus
 }