Merge branch 'main' into refactor/function-ordering-on-branch-protect…

…ion-eval

checks/evaluation/sast.go

@@ -21,34 +21,25 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/finding"
-	"github.com/ossf/scorecard/v4/probes/sastToolCodeQLInstalled"
-	"github.com/ossf/scorecard/v4/probes/sastToolPysaInstalled"
-	"github.com/ossf/scorecard/v4/probes/sastToolQodanaInstalled"
+	"github.com/ossf/scorecard/v4/probes/sastToolConfigured"
 	"github.com/ossf/scorecard/v4/probes/sastToolRunsOnAllCommits"
-	"github.com/ossf/scorecard/v4/probes/sastToolSnykInstalled"
-	"github.com/ossf/scorecard/v4/probes/sastToolSonarInstalled"
 )
 
 // SAST applies the score policy for the SAST check.
 func SAST(name string,
 	findings []finding.Finding, dl checker.DetailLogger,
 ) checker.CheckResult {
-	// We have 3 unique probes, each should have a finding.
 	expectedProbes := []string{
-		sastToolCodeQLInstalled.Probe,
-		sastToolPysaInstalled.Probe,
-		sastToolQodanaInstalled.Probe,
+		sastToolConfigured.Probe,
 		sastToolRunsOnAllCommits.Probe,
-		sastToolSonarInstalled.Probe,
-		sastToolSnykInstalled.Probe,
 	}
 
 	if !finding.UniqueProbesEqual(findings, expectedProbes) {
 		e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")
 		return checker.CreateRuntimeErrorResult(name, e)
 	}
 
-	var sastScore, codeQlScore, pysaScore, qodanaScore, snykScore, sonarScore int
+	var sastScore, codeQlScore, otherScore int
 	var err error
 	// Assign sastScore, codeQlScore and sonarScore
 	for i := range findings {
@@ -59,43 +50,24 @@ func SAST(name string,
 			if err != nil {
 				return checker.CreateRuntimeErrorResult(name, sce.WithMessage(sce.ErrScorecardInternal, err.Error()))
 			}
-		case sastToolCodeQLInstalled.Probe:
-			codeQlScore = getSastToolScore(f, dl)
-		case sastToolSnykInstalled.Probe:
-			snykScore = getSastToolScore(f, dl)
-		case sastToolPysaInstalled.Probe:
-			pysaScore = getSastToolScore(f, dl)
-		case sastToolQodanaInstalled.Probe:
-			qodanaScore = getSastToolScore(f, dl)
-		case sastToolSonarInstalled.Probe:
-			if f.Outcome == finding.OutcomePositive {
-				sonarScore = checker.MaxResultScore
-				dl.Info(&checker.LogMessage{
-					Text:      f.Message,
-					Type:      f.Location.Type,
-					Path:      f.Location.Path,
-					Offset:    *f.Location.LineStart,
-					EndOffset: *f.Location.LineEnd,
-					Snippet:   *f.Location.Snippet,
-				})
-			} else if f.Outcome == finding.OutcomeNegative {
-				sonarScore = checker.MinResultScore
+		case sastToolConfigured.Probe:
+			tool, ok := f.Values[sastToolConfigured.ToolKey]
+			if f.Outcome == finding.OutcomePositive && !ok {
+				return checker.CreateRuntimeErrorResult(name, sce.WithMessage(sce.ErrScorecardInternal, "missing SAST tool"))
+			}
+			score := getSastToolScore(f, dl)
+			switch checker.SASTWorkflowType(tool) {
+			case checker.CodeQLWorkflow:
+				codeQlScore = score
+			default:
+				otherScore = score
 			}
 		}
 	}
 
-	if sonarScore == checker.MaxResultScore {
+	if otherScore == checker.MaxResultScore {
 		return checker.CreateMaxScoreResult(name, "SAST tool detected")
 	}
-	if snykScore == checker.MaxResultScore {
-		return checker.CreateMaxScoreResult(name, "SAST tool detected: Snyk")
-	}
-	if pysaScore == checker.MaxResultScore {
-		return checker.CreateMaxScoreResult(name, "SAST tool detected: Pysa")
-	}
-	if qodanaScore == checker.MaxResultScore {
-		return checker.CreateMaxScoreResult(name, "SAST tool detected: Qodana")
-	}
 
 	if sastScore == checker.InconclusiveResultScore &&
 		codeQlScore == checker.InconclusiveResultScore {

checks/evaluation/sast_test.go

@@ -19,31 +19,21 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/sastToolConfigured"
 	"github.com/ossf/scorecard/v4/probes/sastToolRunsOnAllCommits"
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
 func TestSAST(t *testing.T) {
-	snippet := "some code snippet"
-	sline := uint(10)
-	eline := uint(46)
 	t.Parallel()
 	tests := []struct {
 		name     string
 		findings []finding.Finding
 		result   scut.TestReturn
 	}{
 		{
-			name: "SAST - Missing a probe",
+			name: "SAST - Missing a probe (sastToolConfigured)",
 			findings: []finding.Finding{
-				{
-					Probe:   "sastToolCodeQLInstalled",
-					Outcome: finding.OutcomePositive,
-				},
-				{
-					Probe:   "sastToolSnykInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
 				{
 					Probe:   sastToolRunsOnAllCommits.Probe,
 					Outcome: finding.OutcomePositive,
@@ -55,24 +45,10 @@ func TestSAST(t *testing.T) {
 			},
 		},
 		{
-			name: "Sonar and codeQL is installed. Snyk, Qodana and Pysa are not installed.",
+			name: "Sonar and codeQL is installed",
 			findings: []finding.Finding{
-				{
-					Probe:   "sastToolCodeQLInstalled",
-					Outcome: finding.OutcomePositive,
-				},
-				{
-					Probe:   "sastToolSnykInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolPysaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolQodanaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
+				tool(checker.SonarWorkflow),
+				tool(checker.CodeQLWorkflow),
 				{
 					Probe:   sastToolRunsOnAllCommits.Probe,
 					Outcome: finding.OutcomePositive,
@@ -81,17 +57,6 @@ func TestSAST(t *testing.T) {
 						sastToolRunsOnAllCommits.TotalPRsKey:    "2",
 					},
 				},
-				{
-					Probe:   "sastToolSonarInstalled",
-					Outcome: finding.OutcomePositive,
-					Location: &finding.Location{
-						Type:      finding.FileTypeSource,
-						Path:      "path/to/file.txt",
-						LineStart: &sline,
-						LineEnd:   &eline,
-						Snippet:   &snippet,
-					},
-				},
 			},
 			result: scut.TestReturn{
 				Score:        10,
@@ -102,22 +67,7 @@ func TestSAST(t *testing.T) {
 		{
 			name: "Pysa is installed. CodeQL, Snyk, Qodana and Sonar are not installed.",
 			findings: []finding.Finding{
-				{
-					Probe:   "sastToolCodeQLInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolSnykInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolPysaInstalled",
-					Outcome: finding.OutcomePositive,
-				},
-				{
-					Probe:   "sastToolQodanaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
+				tool(checker.PysaWorkflow),
 				{
 					Probe:   sastToolRunsOnAllCommits.Probe,
 					Outcome: finding.OutcomePositive,
@@ -126,10 +76,6 @@ func TestSAST(t *testing.T) {
 						sastToolRunsOnAllCommits.TotalPRsKey:    "2",
 					},
 				},
-				{
-					Probe:   "sastToolSonarInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
 			},
 			result: scut.TestReturn{
 				Score:        10,
@@ -142,37 +88,11 @@ func TestSAST(t *testing.T) {
 					Does not have info about whether SAST runs
 					on every commit.`,
 			findings: []finding.Finding{
-				{
-					Probe:   "sastToolCodeQLInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolSnykInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolQodanaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolPysaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
+				tool(checker.SonarWorkflow),
 				{
 					Probe:   sastToolRunsOnAllCommits.Probe,
 					Outcome: finding.OutcomeNotApplicable,
 				},
-				{
-					Probe:   "sastToolSonarInstalled",
-					Outcome: finding.OutcomePositive,
-					Location: &finding.Location{
-						Type:      finding.FileTypeSource,
-						Path:      "path/to/file.txt",
-						LineStart: &sline,
-						LineEnd:   &eline,
-						Snippet:   &snippet,
-					},
-				},
 			},
 			result: scut.TestReturn{
 				Score:        10,
@@ -184,19 +104,7 @@ func TestSAST(t *testing.T) {
 			name: "Sonar, CodeQL, Snyk, Qodana and Pysa are not installed",
 			findings: []finding.Finding{
 				{
-					Probe:   "sastToolCodeQLInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolSnykInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolPysaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolQodanaInstalled",
+					Probe:   sastToolConfigured.Probe,
 					Outcome: finding.OutcomeNegative,
 				},
 				{
@@ -207,10 +115,6 @@ func TestSAST(t *testing.T) {
 						sastToolRunsOnAllCommits.TotalPRsKey:    "3",
 					},
 				},
-				{
-					Probe:   "sastToolSonarInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
 			},
 			result: scut.TestReturn{
 				Score:        3,
@@ -221,14 +125,7 @@ func TestSAST(t *testing.T) {
 		{
 			name: "Snyk is installed, Sonar, Qodana and CodeQL are not installed",
 			findings: []finding.Finding{
-				{
-					Probe:   "sastToolCodeQLInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolSnykInstalled",
-					Outcome: finding.OutcomePositive,
-				},
+				tool(checker.SnykWorkflow),
 				{
 					Probe:   sastToolRunsOnAllCommits.Probe,
 					Outcome: finding.OutcomePositive,
@@ -237,18 +134,6 @@ func TestSAST(t *testing.T) {
 						sastToolRunsOnAllCommits.TotalPRsKey:    "3",
 					},
 				},
-				{
-					Probe:   "sastToolSonarInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolPysaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolQodanaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
 			},
 			result: scut.TestReturn{
 				Score:        10,
@@ -259,14 +144,7 @@ func TestSAST(t *testing.T) {
 		{
 			name: "Qodana is installed, Snyk, Sonar, and CodeQL are not installed",
 			findings: []finding.Finding{
-				{
-					Probe:   "sastToolCodeQLInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolSnykInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
+				tool(checker.QodanaWorkflow),
 				{
 					Probe:   sastToolRunsOnAllCommits.Probe,
 					Outcome: finding.OutcomePositive,
@@ -275,18 +153,6 @@ func TestSAST(t *testing.T) {
 						sastToolRunsOnAllCommits.TotalPRsKey:    "3",
 					},
 				},
-				{
-					Probe:   "sastToolSonarInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolPysaInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "sastToolQodanaInstalled",
-					Outcome: finding.OutcomePositive,
-				},
 			},
 			result: scut.TestReturn{
 				Score:        10,
@@ -305,3 +171,13 @@ func TestSAST(t *testing.T) {
 		})
 	}
 }
+
+func tool(name checker.SASTWorkflowType) finding.Finding {
+	return finding.Finding{
+		Probe:   sastToolConfigured.Probe,
+		Outcome: finding.OutcomePositive,
+		Values: map[string]string{
+			sastToolConfigured.ToolKey: string(name),
+		},
+	}
+}

docs/checks/sast/README.md

@@ -0,0 +1,14 @@
+# Supported Tools
+* [CodeQL](https://docs.github.com/code-security/code-scanning/enabling-code-scanning/configuring-default-setup-for-code-scanning)
+  * Detection is based on GitHub workflows using `github/codeql-action/analyze`, or GitHub Action checks run against PRs.
+* [Qodona](https://github.com/JetBrains/qodana-action)
+  * Detection based on GitHub workflows using `JetBrains/qodana-action`.
+* [Snyk](https://github.com/snyk/actions)
+  * Detection based on GitHub workflows using one of the actions from the set at https://github.com/snyk/actions
+* [Sonar](https://docs.sonarsource.com/sonarqube/latest/setup-and-upgrade/overview/)
+  * Detection based on the presence of a `pom.xml` file specifying a `sonar.host.url`, or GitHub Action checks run against PRs.
+
+# Add Support
+
+Don't see your SAST tool listed? 
+Search for an existing issue, or create one, to discuss adding support.