🌱 Combine Dependency-Update-Tool probes into one (#3981)

* add single probe for dependencyUpdateToolConfigured probe Signed-off-by: Spencer Schrock <[email protected]> * delete individual update tool probes Signed-off-by: Spencer Schrock <[email protected]> * use new update tool probe in evaluation Signed-off-by: Spencer Schrock <[email protected]> * fix dependency update tool tests The old test names were unclear, and didn't cover all supported tools. Additionally the warn count changed since there's only one probe now, instead of 3. Signed-off-by: Spencer Schrock <[email protected]> * clarify test name Signed-off-by: Spencer Schrock <[email protected]> --------- Signed-off-by: Spencer Schrock <[email protected]>
ossf · Mar 28, 2024 · 46bb36a · 46bb36a
1 parent 153e06d
commit 46bb36a
Show file tree

Hide file tree

Showing 15 changed files with 216 additions and 722 deletions.
diff --git a/checks/dependency_update_tool_test.go b/checks/dependency_update_tool_test.go
@@ -42,7 +42,7 @@ func TestDependencyUpdateTool(t *testing.T) {
 		wantErr           bool
 	}{
 		{
-			name:    "dependency yml",
+			name:    "dependabot config detected",
 			wantErr: false,
 			files: []string{
 				".github/dependabot.yml",
@@ -55,7 +55,7 @@ func TestDependencyUpdateTool(t *testing.T) {
 			},
 		},
 		{
-			name:    "dependency yaml ",
+			name:    "dependabot alternate yaml extension detected",
 			wantErr: false,
 			files: []string{
 				".github/dependabot.yaml",
@@ -68,66 +68,84 @@ func TestDependencyUpdateTool(t *testing.T) {
 			},
 		},
 		{
-			name:    "foo bar",
+			name:    "renovatebot config detected",
 			wantErr: false,
 			files: []string{
-				".github/foobar.yml",
+				"renovate.json",
 			},
-			SearchCommits:     []clients.Commit{{Committer: clients.User{ID: 111111111}}},
-			CallSearchCommits: 1,
+			CallSearchCommits: 0,
 			expected: scut.TestReturn{
-				NumberOfWarn: 3,
+				NumberOfInfo: 1,
+				NumberOfWarn: 0,
+				Score:        10,
 			},
 		},
 		{
-			name:    "foo bar 2",
+			name:    "alternate renovatebot config detected",
 			wantErr: false,
 			files: []string{
-				".github/foobar.yml",
+				".github/renovate.json5",
 			},
-			SearchCommits:     []clients.Commit{},
-			CallSearchCommits: 1,
+			CallSearchCommits: 0,
 			expected: scut.TestReturn{
-				NumberOfWarn: 3,
+				NumberOfInfo: 1,
+				NumberOfWarn: 0,
+				Score:        10,
 			},
 		},
-
 		{
-			name:    "found in commits",
+			name:    "pyup config detected",
 			wantErr: false,
 			files: []string{
-				".github/foobar.yaml",
+				".pyup.yml",
 			},
-			SearchCommits:     []clients.Commit{{Committer: clients.User{ID: dependabotID}}},
-			CallSearchCommits: 1,
+			CallSearchCommits: 0,
 			expected: scut.TestReturn{
 				NumberOfInfo: 1,
 				NumberOfWarn: 0,
 				Score:        10,
 			},
 		},
 		{
-			name:    "found in commits 2",
+			name:              "random committer ID not detected as dependecy tool bot",
+			wantErr:           false,
+			files:             []string{},
+			SearchCommits:     []clients.Commit{{Committer: clients.User{ID: 111111111}}},
+			CallSearchCommits: 1,
+			expected: scut.TestReturn{
+				NumberOfWarn: 1,
+			},
+		},
+		{
+			name:    "random yaml file not detected as update tool config",
 			wantErr: false,
-			files:   []string{},
-			SearchCommits: []clients.Commit{
-				{Committer: clients.User{ID: 111111111}},
-				{Committer: clients.User{ID: dependabotID}},
+			files: []string{
+				".github/foobar.yml",
 			},
+			SearchCommits:     []clients.Commit{},
+			CallSearchCommits: 1,
+			expected: scut.TestReturn{
+				NumberOfWarn: 1,
+			},
+		},
+		{
+			name:    "dependabot found in recent commits",
+			wantErr: false,
+			files: []string{
+				".github/foobar.yaml",
+			},
+			SearchCommits:     []clients.Commit{{Committer: clients.User{ID: dependabotID}}},
 			CallSearchCommits: 1,
 			expected: scut.TestReturn{
 				NumberOfInfo: 1,
 				NumberOfWarn: 0,
 				Score:        10,
 			},
 		},
-
 		{
-			name:    "many commits",
+			name:    "dependabot bot found in recent commits 2",
 			wantErr: false,
-			files: []string{
-				".github/foobar.yml",
-			},
+			files:   []string{},
 			SearchCommits: []clients.Commit{
 				{Committer: clients.User{ID: 111111111}},
 				{Committer: clients.User{ID: dependabotID}},

diff --git a/checks/evaluation/dependency_update_tool.go b/checks/evaluation/dependency_update_tool.go
@@ -18,9 +18,7 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/finding"
-	"github.com/ossf/scorecard/v4/probes/toolDependabotInstalled"
-	"github.com/ossf/scorecard/v4/probes/toolPyUpInstalled"
-	"github.com/ossf/scorecard/v4/probes/toolRenovateInstalled"
+	"github.com/ossf/scorecard/v4/probes/dependencyUpdateToolConfigured"
 )
 
 // DependencyUpdateTool applies the score policy and logs the details
@@ -29,9 +27,7 @@ func DependencyUpdateTool(name string,
 	findings []finding.Finding, dl checker.DetailLogger,
 ) checker.CheckResult {
 	expectedProbes := []string{
-		toolDependabotInstalled.Probe,
-		toolPyUpInstalled.Probe,
-		toolRenovateInstalled.Probe,
+		dependencyUpdateToolConfigured.Probe,
 	}
 	if !finding.UniqueProbesEqual(findings, expectedProbes) {
 		e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")

diff --git a/checks/evaluation/dependency_update_tool_test.go b/checks/evaluation/dependency_update_tool_test.go
@@ -20,6 +20,7 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/dependencyUpdateToolConfigured"
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
@@ -31,123 +32,44 @@ func TestDependencyUpdateTool(t *testing.T) {
 		result   scut.TestReturn
 	}{
 		{
-			name: "dependabot",
+			name: "one update tool is max score",
 			findings: []finding.Finding{
-				{
-					Probe:   "toolDependabotInstalled",
-					Outcome: finding.OutcomePositive,
-				},
-				{
-					Probe:   "toolPyUpInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolRenovateInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
+				depUpdateTool("Dependabot"),
 			},
 			result: scut.TestReturn{
 				Score:        checker.MaxResultScore,
 				NumberOfInfo: 1,
 			},
 		},
 		{
-			name: "renovate",
+			name: "multiple update tools both logged",
 			findings: []finding.Finding{
-				{
-					Probe:   "toolDependabotInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolPyUpInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolRenovateInstalled",
-					Outcome: finding.OutcomePositive,
-				},
+				depUpdateTool("RenovateBot"),
+				depUpdateTool("PyUp"),
 			},
 			result: scut.TestReturn{
 				Score:        checker.MaxResultScore,
-				NumberOfInfo: 1,
+				NumberOfInfo: 2,
 			},
 		},
 		{
-			name: "pyup",
+			name: "no update tool is min score",
 			findings: []finding.Finding{
 				{
-					Probe:   "toolDependabotInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolPyUpInstalled",
-					Outcome: finding.OutcomePositive,
-				},
-				{
-					Probe:   "toolRenovateInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-			},
-			result: scut.TestReturn{
-				Score:        checker.MaxResultScore,
-				NumberOfInfo: 1,
-			},
-		},
-		{
-			name: "none",
-			findings: []finding.Finding{
-				{
-					Probe:   "toolDependabotInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolRenovateInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolPyUpInstalled",
+					Probe:   dependencyUpdateToolConfigured.Probe,
 					Outcome: finding.OutcomeNegative,
 				},
 			},
 			result: scut.TestReturn{
 				Score:        checker.MinResultScore,
-				NumberOfWarn: 3,
+				NumberOfWarn: 1,
 			},
 		},
 		{
-			name: "missing probes renovate",
+			name: "invalid probe name is an error",
 			findings: []finding.Finding{
 				{
-					Probe:   "toolDependabotInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolPyUpInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-			},
-			result: scut.TestReturn{
-				Score: checker.InconclusiveResultScore,
-				Error: sce.ErrScorecardInternal,
-			},
-		},
-		{
-			name: "invalid probe name",
-			findings: []finding.Finding{
-				{
-					Probe:   "toolDependabotInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolRenovateInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolPyUpInstalled",
-					Outcome: finding.OutcomeNegative,
-				},
-				{
-					Probe:   "toolInvalidProbeName",
+					Probe:   "notARealProbe",
 					Outcome: finding.OutcomeNegative,
 				},
 			},
@@ -161,10 +83,19 @@ func TestDependencyUpdateTool(t *testing.T) {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-
 			dl := scut.TestDetailLogger{}
 			got := DependencyUpdateTool(tt.name, tt.findings, &dl)
 			scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl)
 		})
 	}
 }
+
+func depUpdateTool(name string) finding.Finding {
+	return finding.Finding{
+		Probe:   dependencyUpdateToolConfigured.Probe,
+		Outcome: finding.OutcomePositive,
+		Values: map[string]string{
+			dependencyUpdateToolConfigured.ToolKey: name,
+		},
+	}
+}
diff --git a/docs/checks/dependencyupdatetool/README.md b/docs/checks/dependencyupdatetool/README.md
@@ -0,0 +1,12 @@
+# Supported Tools
+* [Dependabot](https://docs.github.com/code-security/getting-started/dependabot-quickstart-guide)
+  * Detection is based on a `.github/dependabot.yml` or `.github/dependabot.yaml` file.
+* [Renovate](https://docs.renovatebot.com/)
+  * Detection is based on the configuration files listed [here](https://docs.renovatebot.com/configuration-options/)
+* [PyUp](https://github.com/pyupio/pyup)
+  * Detection based on a `.pyup.yml` file
+
+# Add Support
+
+Don't see your update tool listed?
+Search for an existing issue, or create one, to discuss adding support.
diff --git a/probes/toolRenovateInstalled/def.yml → ...es/dependencyUpdateToolConfigured/def.yml b/probes/toolRenovateInstalled/def.yml → ...es/dependencyUpdateToolConfigured/def.yml
@@ -1,4 +1,4 @@
-# Copyright 2023 OpenSSF Scorecard Authors
+# Copyright 2024 OpenSSF Scorecard Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,24 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-id: toolRenovateInstalled
-short: Check that Renovate bot is installed.
+id: dependencyUpdateToolConfigured
+short: Check that a dependency update tool config is present.
 motivation: >
   Out-of-date dependencies make a project vulnerable to known flaws and prone to attacks.
-  Renovate automates the process of updating dependencies by scanning for outdated or insecure requirements, and opening a pull request to update them if found.
+  Tools can automate the process of updating dependencies by scanning for outdated or insecure requirements, and opening a pull request to update them if found.
 implementation: >
-  The implementation looks for the presence of files named ".github/renovate.json", ".github/renovate.json5", ".renovaterc.json" or. "renovate.json".
-  If none of these files are found, Renovate is not installed.
-  NOTE: the implementation does not ensure that Renovate is run or that Renovate's pull requests are merged.
+  The implementation looks for the presence of various config files for different dependency update tools.
 outcome:
-  - If Renovate is installed, the probe returns OutcomePositive (1) for each configuration.
-  - If Renovate is not installed, the probe returns OutcomeNegative (0).
+  - If a dependency update tool is configured, the probe returns OutcomePositive for each configuration.
+  - If no tool is detected, the probe returns OutcomeNegative.
 remediation:
   effort: Low
   text:
-    - Follow the instructions from https://docs.renovatebot.com/configuration-options/.
+    - Setup one of tools we currently detect https://github.com/ossf/scorecard/blob/main/docs/checks/dependencyupdatetool/README.md.
   markdown:
-    - Follow the instructions from [the official documentation](https://docs.renovatebot.com/configuration-options/).
+    - Setup one of [tools we currently detect](https://github.com/ossf/scorecard/blob/main/docs/checks/dependencyupdatetool/README.md).
 ecosystem:
   languages:
     - dockerfile
@@ -42,4 +40,5 @@ ecosystem:
     - rust
   clients:
     - github
-    - gitlab
+    - gitlab
+    - localdir