Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🌱 use ValidateTestReturn for SAST tests #3872

Merged
merged 3 commits into from
Feb 13, 2024
Merged

🌱 use ValidateTestReturn for SAST tests #3872

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3c7b83f
check logger counts for SAST tests
spencerschrock Feb 12, 2024
b367c26
clarify test names and score constants used
spencerschrock Feb 12, 2024
56ae55a
Merge branch 'main' into sast-compare-logger-messages
spencerschrock Feb 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 45 additions & 24 deletions checks/sast_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/clients"
mockrepo "github.com/ossf/scorecard/v4/clients/mockclients"
sce "github.com/ossf/scorecard/v4/errors"
scut "github.com/ossf/scorecard/v4/utests"
)

Expand All @@ -39,24 +40,31 @@ func Test_SAST(t *testing.T) {
searchRequest clients.SearchRequest
name string
path string
expected checker.CheckResult
commits []clients.Commit
checkRuns []clients.CheckRun
searchresult clients.SearchResponse
expected scut.TestReturn
}{
{
name: "SAST checker should return failed status when no PRs are found",
name: "SAST checker should return min score when no PRs are found",
commits: []clients.Commit{},
searchresult: clients.SearchResponse{},
checkRuns: []clients.CheckRun{},
expected: scut.TestReturn{
Score: checker.MinResultScore,
NumberOfWarn: 1,
},
},
{
name: "SAST checker should return failed status when no PRs are found",
name: "SAST checker should return failed status when an error occurs",
err: errors.New("error"),
commits: []clients.Commit{},
searchresult: clients.SearchResponse{},
checkRuns: []clients.CheckRun{},
expected: checker.CheckResult{Score: -1},
expected: scut.TestReturn{
Score: checker.InconclusiveResultScore,
Error: sce.ErrScorecardInternal,
},
},
{
name: "Successful SAST checker should return success status for github-advanced-security",
Expand All @@ -77,8 +85,10 @@ func Test_SAST(t *testing.T) {
},
},
},
expected: checker.CheckResult{
Score: 10,
expected: scut.TestReturn{
Score: checker.MaxResultScore,
NumberOfInfo: 1,
NumberOfDebug: 1,
},
},
{
Expand All @@ -100,8 +110,10 @@ func Test_SAST(t *testing.T) {
},
},
},
expected: checker.CheckResult{
Score: 10,
expected: scut.TestReturn{
Score: checker.MaxResultScore,
NumberOfInfo: 1,
NumberOfDebug: 1,
},
},
{
Expand All @@ -124,8 +136,10 @@ func Test_SAST(t *testing.T) {
},
},
path: "",
expected: checker.CheckResult{
Score: 10,
expected: scut.TestReturn{
Score: checker.MaxResultScore,
NumberOfInfo: 1,
NumberOfDebug: 1,
},
},
{
Expand All @@ -147,8 +161,10 @@ func Test_SAST(t *testing.T) {
},
},
},
expected: checker.CheckResult{
Score: 10,
expected: scut.TestReturn{
Score: checker.MaxResultScore,
NumberOfInfo: 1,
NumberOfDebug: 1,
},
},
{
Expand All @@ -163,8 +179,10 @@ func Test_SAST(t *testing.T) {
},
searchresult: clients.SearchResponse{},
path: ".github/workflows/airflow-codeql-workflow.yaml",
expected: checker.CheckResult{
Score: 7,
expected: scut.TestReturn{
Score: 7,
NumberOfWarn: 1,
NumberOfInfo: 1,
},
},
{
Expand Down Expand Up @@ -195,8 +213,10 @@ func Test_SAST(t *testing.T) {
},
},
path: ".github/workflows/airflow-codeql-workflow.yaml",
expected: checker.CheckResult{
Score: 10,
expected: scut.TestReturn{
Score: checker.MaxResultScore,
NumberOfInfo: 2,
NumberOfDebug: 1,
},
},
{
Expand Down Expand Up @@ -228,8 +248,10 @@ func Test_SAST(t *testing.T) {
},
},
path: ".github/workflows/airflow-codeql-workflow.yaml",
expected: checker.CheckResult{
Score: 10,
expected: scut.TestReturn{
Score: checker.MaxResultScore,
NumberOfInfo: 2,
NumberOfDebug: 1,
},
},
{
Expand Down Expand Up @@ -266,8 +288,10 @@ func Test_SAST(t *testing.T) {
},
},
path: ".github/workflows/airflow-codeql-workflow.yaml",
expected: checker.CheckResult{
Score: 7,
expected: scut.TestReturn{
Score: 7,
NumberOfWarn: 1,
NumberOfInfo: 1,
},
},
}
Expand Down Expand Up @@ -315,10 +339,7 @@ func Test_SAST(t *testing.T) {
}
res := SAST(&req)

if res.Score != tt.expected.Score {
t.Errorf("Expected score %d, got %d for %v", tt.expected.Score, res.Score, tt.name)
}
ctrl.Finish()
spencerschrock marked this conversation as resolved.
Show resolved Hide resolved
scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl)
})
}
}
Loading