Align the latest OSCAL Assessment Results and consolidate ocm test da…

…ta in ocm dedicated directory

Signed-off-by: Takumi Yanagawa <[email protected]>

cmd/ocm/oscal2policy/c2p-config.yaml

@@ -7,7 +7,7 @@ compliance:
   componentDefinition:
     url: ./pkg/composer/testdata/oscal/component-definition.json
 policyResources: 
-  url: ./pkg/composer/testdata/policies
+  url: ./pkg/composer/testdata/ocm/policies
 policyResults: 
   url: a/b/c
 clusterGroups:

cmd/ocm/result2oscal/c2p-config.yaml

@@ -7,9 +7,9 @@ compliance:
   componentDefinition:
     url: ./pkg/testdata/oscal/reporter-test/component-definition.json
 policyResources: 
-  url: ./pkg/testdata/policies
+  url: ./pkg/testdata/ocm/policies
 policyResults: 
-  url: ./pkg/testdata/policy-results
+  url: ./pkg/testdata/ocm/policy-results
 clusterGroups:
   - name: test-group # name of clusterGroup
     matchLabels:

cmd/ocm/result2oscal/cmd/cmd.go

@@ -33,7 +33,7 @@ func New() *cobra.Command {
 
 	command := &cobra.Command{
 		Use:   "result2oscal",
-		Short: "Generate OSCAL Assessment Results from Kyverno policies and the policy reports",
+		Short: "Generate OSCAL Assessment Results from OCM Policy statuses",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := opts.Complete(); err != nil {
 				return err
@@ -71,26 +71,15 @@ func Run(options *options.Options) error {
 
 	r := reporter.NewReporter(c2pcrParsed)
 	r.SetGenerationType(reporter.GenerationTypePolicyReport)
-	report, err := r.Generate()
+	arRoot, err := r.Generate()
 	if err != nil {
 		panic(err)
 	}
 
-	err = pkg.WriteObjToYamlFile(outputDir+"/compliance-report.yaml", report)
+	err = pkg.WriteObjToJsonFile(outputDir+"/assessment-results.json", arRoot)
 	if err != nil {
 		panic(err)
 	}
 
-	for _, pr := range r.GetPolicyReports() {
-		nspath, err := pkg.MakeDir(outputDir + "/" + pr.Namespace)
-		if err != nil {
-			panic(err)
-		}
-		err = pkg.WriteObjToYamlFile(nspath+"/"+pr.Name+".yaml", pr)
-		if err != nil {
-			panic(err)
-		}
-	}
-
 	return nil
 }

docs/oscal-vs-ocm-status-mapping.csv

@@ -0,0 +1,16 @@
+Field of OSCAL Assessment Result,How is OCM Policy status mapped,Field of OCM Policy status
+local-definitions.inventory-items[],Per cluster,status_status = policies[].find(x -> x.namespace == hub ns).flatmap(x1 -> status.status[].map(x2 -> x2))
+local-definitions.inventory-items[].props[].(cluster-name),Cluster name,status_status.clustername
+observations[],Per rule (= per policy),OSCAL CDef Rule Ids
+observations[].props[].(assessment-rule-id),Rule Id,OSCAL CDef Rule Id
+observations[].props[].(policy-id),Policy Id,OSCAL CDef Policy Id
+observations[].props[].(control-id),Control Id,OSCAL CDef Control Id
+observations[].props[].(result),Aggregated status of compliance over clusters (=subjects),policies[].find(x -> x.namespace == hub ns && x -> x.metadata.name == policy-id).status.compliant == “Compliant” ? “pass” : “fail”
+observations[].subjects[],Per policy status of cluster,status[i] := policies[].find(x -> x.metadata.name == policy-id).status.status[]
+observations[].subjects[].subject-uuid,Inventory item id of the check,inventory-items[].find(x -> x.props[].(cluster-name) == status[i].clustername).id
+observations[].subjects[].props[].(result),Status of compliance (pass/fail),status[i].compliant == “Compliant” ? “pass” : “fail”
+observations[].subjects[].props[].(reason),Reason of the status,details := policies[].find(x -> x.namespace == clustername && x.metadata.name == policy-id).status.details
+observations[].subjects[].props[].(evaluate-on),Evaluation timestamp,details.map(x -> x.history[0].lastTimestamp).sort_by_descend()[0]
+observations[].relevant_references[].description,Evidence description,TBD
+observations[].relevant_references[].href,Evidence urls,TBD
+observations[].links[].href,Report urls,TBD

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 require (
 	github.com/go-logr/logr v1.2.4
+	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/kcp-dev/kcp/pkg/apis v0.11.0
 	github.com/kyverno/kyverno v1.10.3
@@ -128,7 +129,6 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/certificate-transparency-go v1.1.4 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-containerregistry v0.14.0 // indirect
 	github.com/google/go-github/v45 v45.2.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect

pkg/ocm/oscal2policy_test.go

@@ -27,11 +27,10 @@ import (
 )
 
 func TestOscal2Policy(t *testing.T) {
-	policyDir := pkg.PathFromPkgDirectory("./testdata/policies")
+	policyDir := pkg.PathFromPkgDirectory("./testdata/ocm/policies")
 	catalogPath := pkg.PathFromPkgDirectory("./testdata/oscal/catalog.json")
 	profilePath := pkg.PathFromPkgDirectory("./testdata/oscal/profile.json")
-	cdPath := pkg.PathFromPkgDirectory("./testdata/oscal/component-definition.json")
-	// expectedDir := pkg.PathFromPkgDirectory("./composer/testdata/expected/c2pcr-parser-composed-policies")
+	cdPath := pkg.PathFromPkgDirectory("./testdata/ocm/component-definition.json")
 
 	tempDirPath := pkg.PathFromPkgDirectory("./testdata/_test")
 	err := os.MkdirAll(tempDirPath, os.ModePerm)

pkg/ocm/reporter/helper_test.go

@@ -36,7 +36,7 @@ func TestConvertToPolicyReport(t *testing.T) {
 		func(policySet typepolicy.PolicySet) {},
 		func(placementDecision typeplacementdecision.PlacementDecision) {},
 	)
-	err := filepath.Walk(pkg.PathFromPkgDirectory("./testdata/policy-results"), traverseFunc)
+	err := filepath.Walk(pkg.PathFromPkgDirectory("./testdata/ocm/policy-results"), traverseFunc)
 	assert.NoError(t, err, "Should not happen")
 
 	tempDirPath := pkg.PathFromPkgDirectory("./testdata/_test")

pkg/ocm/reporter/reporter.go

@@ -32,6 +32,8 @@ import (
 
 	"github.com/IBM/compliance-to-policy/pkg/oscal"
 	typec2pcr "github.com/IBM/compliance-to-policy/pkg/types/c2pcr"
+	typear "github.com/IBM/compliance-to-policy/pkg/types/oscal/assessmentresults"
+	typeoscalcommon "github.com/IBM/compliance-to-policy/pkg/types/oscal/common"
 	typeplacementdecision "github.com/IBM/compliance-to-policy/pkg/types/placementdecision"
 	typepolicy "github.com/IBM/compliance-to-policy/pkg/types/policy"
 	typereport "github.com/IBM/compliance-to-policy/pkg/types/report"
@@ -79,11 +81,7 @@ func (r *Reporter) SetGenerationType(generationType GenerationType) {
 	r.generationType = generationType
 }
 
-func (r *Reporter) GetPolicyReports() []*typepolr.PolicyReport {
-	return r.policyReports
-}
-
-func (r *Reporter) Generate() (typereport.ComplianceReport, error) {
+func (r *Reporter) Generate() (*typear.AssessmentResultsRoot, error) {
 	traverseFunc := genTraverseFunc(
 		func(policy typepolicy.Policy) { r.policies = append(r.policies, &policy) },
 		func(policySet typepolicy.PolicySet) { r.policySets = append(r.policySets, &policySet) },
@@ -94,11 +92,30 @@ func (r *Reporter) Generate() (typereport.ComplianceReport, error) {
 	if err := filepath.Walk(r.c2pParsed.PolicyResultsDir, traverseFunc); err != nil {
 		logger.Error(err.Error())
 	}
+
+	inventories := []typear.InventoryItem{}
+	clusternameIndex := map[string]bool{}
 	for _, policy := range r.policies {
 		polr := ConvertToPolicyReport(*policy)
 		r.policyReports = append(r.policyReports, &polr)
+		if policy.Namespace == r.c2pParsed.Namespace {
+			for _, s := range policy.Status.Status {
+				_, exist := clusternameIndex[s.ClusterName]
+				if !exist {
+					clusternameIndex[s.ClusterName] = true
+					item := typear.InventoryItem{
+						UUID: oscal.GenerateUUID(),
+						Props: []typeoscalcommon.Prop{{
+							Name:  "cluster-name",
+							Value: s.ClusterName,
+						}},
+					}
+					inventories = append(inventories, item)
+				}
+			}
+		}
 	}
-	reportComponents := []typereport.Component{}
+	observations := []typear.Observation{}
 	for _, cdobj := range r.c2pParsed.ComponentObjects {
 		policySets := typeutils.FilterByAnnotation(r.policySets, pkg.ANNOTATION_COMPONENT_TITLE, cdobj.ComponentTitle)
 		clusterNameSets := sets.NewString()
@@ -119,7 +136,6 @@ func (r *Reporter) Generate() (typereport.ComplianceReport, error) {
 			}
 		}
 		for _, controlImpleObj := range cdobj.ControlImpleObjects {
-			controlResults := []typereport.ControlResult{}
 			requiredControls := sets.NewString()
 			checkedControls := sets.NewString()
 			for _, controlObj := range controlImpleObj.ControlObjects {
@@ -140,71 +156,107 @@ func (r *Reporter) Generate() (typereport.ComplianceReport, error) {
 						if policySet != nil {
 							policy = typeutils.FindByNamespaceName(r.policies, policySet.Namespace, policyId)
 						}
-						var reason string
 						var ruleStatus typereport.RuleStatus
+						subjects := []typear.Subject{}
 						if policy != nil {
 							var reasons []Reason
 							if r.generationType == GenerationTypePolicyReport {
 								reasons = r.GenerateReasonsFromPolicyReports(*policy)
 							} else {
 								reasons = r.GenerateReasonsFromRawPolicies(*policy)
 							}
-							if statusByte, err := sigyaml.Marshal(reasons); err == nil {
-								reason = string(statusByte)
-							} else {
-								reason = err.Error()
-							}
 							ruleStatus = mapToRuleStatus(policy.Status.ComplianceState)
+							for _, reason := range reasons {
+								var clusterName string
+								var inventoryUuid string
+								for _, inventory := range inventories {
+									prop, ok := oscal.FindProp("cluster-name", inventory.Props)
+									if ok {
+										clusterName = prop.Value
+										inventoryUuid = inventory.UUID
+									} else {
+										clusterName = "N/A"
+										inventoryUuid = ""
+									}
+								}
+								if inventoryUuid != "" {
+									var message string
+									if messageByte, err := sigyaml.Marshal(reason.Messages); err == nil {
+										message = string(messageByte)
+									} else {
+										message = err.Error()
+									}
+									subject := typear.Subject{
+										SubjectUUID: inventoryUuid,
+										Type:        "resource",
+										Title:       "Cluster Name: " + clusterName,
+										Props: []typeoscalcommon.Prop{{
+											Name:  "result",
+											Value: string(mapToRuleStatus(reason.ComplianceState)),
+										}, {
+											Name:  "reason",
+											Value: message,
+										}},
+									}
+									subjects = append(subjects, subject)
+								}
+							}
 						} else {
-							reason = fmt.Sprintf("Unable to find policy status for policy %s", policyId)
 							ruleStatus = typereport.RuleStatusError
 						}
-						ruleResult := typereport.RuleResult{
-							RuleId:   ruleId,
-							PolicyId: policyId,
-							Status:   ruleStatus,
-							Reason:   reason,
+						observation := typear.Observation{
+							UUID:        oscal.GenerateUUID(),
+							Description: fmt.Sprintf("Observation of policy %s", policyId),
+							Methods:     []string{"TEST-AUTOMATED"},
+							Props: []typeoscalcommon.Prop{{
+								Name:  "assessment-rule-id",
+								Value: ruleId,
+							}, {
+								Name:  "policy-id",
+								Value: policyId,
+							}, {
+								Name:  "control-id",
+								Value: controlId,
+							}, {
+								Name:  "result",
+								Value: string(ruleStatus),
+							}},
+							Subjects: subjects,
 						}
-						ruleResults = append(ruleResults, ruleResult)
+						observations = append(observations, observation)
 						checkedControls.Insert(controlId)
 					}
 				}
-				controlResult := typereport.ControlResult{
-					ControlId:        controlId,
-					RuleResults:      ruleResults,
-					ComplianceStatus: aggregateRuleResults(ruleResults),
-				}
-				controlResults = append(controlResults, controlResult)
-			}
-			parameters := map[string]string{}
-			for _, setParam := range controlImpleObj.SetParameters {
-				parameters[setParam.ParamID] = setParam.Values[0]
-			}
-			reportComponent := typereport.Component{
-				ComponentTitle:   cdobj.ComponentTitle,
-				RequiredControls: requiredControls.List(),
-				CheckedControls:  checkedControls.List(),
-				Parameters:       parameters,
-				ControlResults:   controlResults,
-				ComplianceStatus: aggregateControlResults(controlResults),
 			}
-			reportComponents = append(reportComponents, reportComponent)
 		}
 	}
-	spec := typereport.Spec{
-		Catalog:    r.c2pParsed.Catalog.Metadata.Title,
-		Profile:    r.c2pParsed.Profile.Metadata.Title,
-		Components: reportComponents,
+
+	metadata := typear.Metadata{
+		Title:        "OSCAL Assessment Results",
+		LastModified: time.Now(),
+		Version:      "0.0.1",
+		OscalVersion: "1.0.4",
 	}
-	complianceReport := typereport.ComplianceReport{
-		ObjectMeta: v1.ObjectMeta{
-			Name:              "compliance-report",
-			CreationTimestamp: v1.Now(),
-		},
-		Spec: spec,
+	importAp := typear.ImportAp{
+		Href: "http://...",
+	}
+	ar := typear.AssessmentResults{
+		UUID:     oscal.GenerateUUID(),
+		Metadata: metadata,
+		ImportAp: importAp,
+		Results:  []typear.Result{},
+	}
+	result := typear.Result{
+		UUID:         oscal.GenerateUUID(),
+		Title:        "Assessment Results by OCM",
+		Description:  "Assessment Results by OCM...",
+		Start:        time.Now(),
+		Observations: observations,
 	}
+	ar.Results = append(ar.Results, result)
+	arRoot := typear.AssessmentResultsRoot{AssessmentResults: ar}
 
-	return complianceReport, nil
+	return &arRoot, nil
 }
 
 func (r *Reporter) GenerateReasonsFromRawPolicies(policy typepolicy.Policy) []Reason {
@@ -275,46 +327,6 @@ func aggregatePolicyReportSummaryToComplianceState(summary typepolr.PolicyReport
 	}
 }
 
-func aggregateRuleResults(ruleResults []typereport.RuleResult) typereport.ComplianceStatus {
-	countPass := 0
-	countFail := 0
-	countError := 0
-	countUnimple := 0
-	for _, ruleResult := range ruleResults {
-		switch ruleResult.Status {
-		case typereport.RuleStatusPass:
-			countPass++
-		case typereport.RuleStatusFail:
-			countFail++
-		case typereport.RuleStatusError:
-			countError++
-		case typereport.RuleStatusUnImplemented:
-			countUnimple++
-		}
-	}
-	if countPass != 0 && countPass == len(ruleResults) {
-		return typereport.ComplianceStatusCompliant
-	}
-	return typereport.ComplianceStatusNonCompliant
-}
-
-func aggregateControlResults(controlResults []typereport.ControlResult) typereport.ComplianceStatus {
-	countCompiant := 0
-	countNonCompiant := 0
-	for _, controlResult := range controlResults {
-		switch controlResult.ComplianceStatus {
-		case typereport.ComplianceStatusCompliant:
-			countCompiant++
-		case typereport.ComplianceStatusNonCompliant:
-			countNonCompiant++
-		}
-	}
-	if countCompiant != 0 && countCompiant == len(controlResults) {
-		return typereport.ComplianceStatusCompliant
-	}
-	return typereport.ComplianceStatusNonCompliant
-}
-
 func genTraverseFunc(onPolicy func(typepolicy.Policy), onPolicySet func(typepolicy.PolicySet), onPlacementDesicion func(typeplacementdecision.PlacementDecision)) func(path string, info os.FileInfo, err error) error {
 	return func(path string, info os.FileInfo, err error) error {
 		if err != nil {