✨ scdiff: normalize scorecard results (ossf#3294)

* Normalize results.

Signed-off-by: Spencer Schrock <[email protected]>

* add tests for standardized output format

Signed-off-by: Spencer Schrock <[email protected]>

* move comment to proper place.

Signed-off-by: Spencer Schrock <[email protected]>

* add comment about pretty print.

Signed-off-by: Spencer Schrock <[email protected]>

* format test can be parallel.

Signed-off-by: Spencer Schrock <[email protected]>

* Add normalize nil test.

Signed-off-by: Spencer Schrock <[email protected]>

---------

Signed-off-by: Spencer Schrock <[email protected]>
Signed-off-by: Allen Shearin <[email protected]>

cmd/internal/scdiff/app/format/format.go

@@ -15,20 +15,46 @@
 package format
 
 import (
-	"os"
+	"io"
+	"sort"
+	"time"
 
 	"github.com/ossf/scorecard/v4/docs/checks"
 	"github.com/ossf/scorecard/v4/log"
 	"github.com/ossf/scorecard/v4/pkg"
 )
 
+const logLevel = log.DefaultLevel
+
+func normalize(r *pkg.ScorecardResult) {
+	if r == nil {
+		return
+	}
+
+	// these fields will change run-to-run, and aren't indicative of behavior changes.
+	r.Repo.CommitSHA = ""
+	r.Scorecard = pkg.ScorecardInfo{}
+	r.Date = time.Time{}
+
+	sort.Slice(r.Checks, func(i, j int) bool {
+		return r.Checks[i].Name < r.Checks[j].Name
+	})
+
+	for i := range r.Checks {
+		check := &r.Checks[i]
+		sort.Slice(check.Details, func(i, j int) bool {
+			return pkg.DetailToString(&check.Details[i], logLevel) < pkg.DetailToString(&check.Details[j], logLevel)
+		})
+	}
+}
+
 //nolint:wrapcheck
-func JSON(r *pkg.ScorecardResult) error {
+func JSON(r *pkg.ScorecardResult, w io.Writer) error {
 	const details = true
 	docs, err := checks.Read()
 	if err != nil {
 		return err
 	}
-	// TODO standardize the input, and output it to a file
-	return r.AsJSON2(details, log.DefaultLevel, docs, os.Stdout)
+	normalize(r)
+	return r.AsJSON2(details, logLevel, docs, w)
 }

cmd/internal/scdiff/app/format/format_test.go

@@ -0,0 +1,175 @@
+// Copyright 2023 OpenSSF Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package format
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/ossf/scorecard/v4/checker"
+	"github.com/ossf/scorecard/v4/pkg"
+)
+
+func TestJSON(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name string
+		a, b pkg.ScorecardResult
+	}{
+		{
+			name: "repo commit SHA standardized",
+			a: pkg.ScorecardResult{
+				Repo: pkg.RepoInfo{
+					Name:      "github.com/foo/bar",
+					CommitSHA: "commit a",
+				},
+			},
+			b: pkg.ScorecardResult{
+				Repo: pkg.RepoInfo{
+					Name:      "github.com/foo/bar",
+					CommitSHA: "commit b",
+				},
+			},
+		},
+		{
+			name: "dates standardized",
+			a: pkg.ScorecardResult{
+				Date: time.Now(),
+			},
+			b: pkg.ScorecardResult{
+				Date: time.Now().AddDate(0, 0, -1),
+			},
+		},
+		{
+			name: "scorecard info standardized",
+			a: pkg.ScorecardResult{
+				Scorecard: pkg.ScorecardInfo{
+					Version:   "version a",
+					CommitSHA: "scorecard commit x",
+				},
+			},
+			b: pkg.ScorecardResult{
+				Scorecard: pkg.ScorecardInfo{
+					Version:   "version b",
+					CommitSHA: "scorecard commit y",
+				},
+			},
+		},
+		{
+			name: "check order standardized",
+			a: pkg.ScorecardResult{
+				Checks: []checker.CheckResult{
+					{
+						Name:  "Token-Permissions",
+						Score: 10,
+					},
+					{
+						Name:  "License",
+						Score: 10,
+					},
+				},
+			},
+			b: pkg.ScorecardResult{
+				Checks: []checker.CheckResult{
+					{
+						Name:  "License",
+						Score: 10,
+					},
+					{
+						Name:  "Token-Permissions",
+						Score: 10,
+					},
+				},
+			},
+		},
+		{
+			name: "detail order standardized",
+			a: pkg.ScorecardResult{
+				Checks: []checker.CheckResult{
+					{
+						Name:  "Token-Permissions",
+						Score: 10,
+						Details: []checker.CheckDetail{
+							{
+								Msg: checker.LogMessage{
+									Text: "foo",
+								},
+								Type: checker.DetailInfo,
+							},
+							{
+								Msg: checker.LogMessage{
+									Text: "bar",
+								},
+								Type: checker.DetailWarn,
+							},
+						},
+					},
+				},
+			},
+			b: pkg.ScorecardResult{
+				Checks: []checker.CheckResult{
+					{
+						Name:  "Token-Permissions",
+						Score: 10,
+						Details: []checker.CheckDetail{
+							{
+								Msg: checker.LogMessage{
+									Text: "bar",
+								},
+								Type: checker.DetailWarn,
+							},
+							{
+								Msg: checker.LogMessage{
+									Text: "foo",
+								},
+								Type: checker.DetailInfo,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			var bufA, bufB bytes.Buffer
+			err := JSON(&tt.a, &bufA)
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+			err = JSON(&tt.b, &bufB)
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+			if bufA.String() != bufB.String() {
+				t.Errorf("outputs not identical: %s", cmp.Diff(bufA.String(), bufB.String()))
+			}
+		})
+	}
+}
+
+func Test_normalize_nil_safe(t *testing.T) {
+	var x, y *pkg.ScorecardResult
+	normalize(x)
+	normalize(y)
+	if !cmp.Equal(x, y) {
+		t.Errorf("normalized results differ: %v", cmp.Diff(x, y))
+	}
+}

cmd/internal/scdiff/app/generate.go

@@ -50,7 +50,8 @@ var (
 				if err != nil {
 					return fmt.Errorf("running scorecard on %s: %w", scanner.Text(), err)
 				}
-				err = format.JSON(&results)
+				// TODO output it to a file, preferably in pretty printed json
+				err = format.JSON(&results, os.Stdout)
 				if err != nil {
 					return fmt.Errorf("formatting results: %w", err)
 				}