Skip to content

Commit

Permalink
CI-Test: stop assuming either "statuses" or "check runs" are used (#1259
Browse files Browse the repository at this point in the history 
)

Projects with a lot of different CI services use both and the check
should take that into account so as not to report that PRs
like systemd/systemd#21329
with 28 successful, 4 failing, and 2 neutral checks were merged
without any tests.

Without this patch `scorecard` says that 5 out 30 PRs were merged
without running tests:
```
        "Debug: CI test found: pr: 21299, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/472a52d22b9f505a4c284e7ad020ea1b3728466d",
        "Debug: CI test found: pr: 21300, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/cf35602cbc1bffcb5ab01153af153d0bf4e5a7e9",
        "Debug: CI test found: pr: 21301, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/13b7b8bd7362d198e123a67b8dfdc1c558df5b21",
        "Debug: CI test found: pr: 21302, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/dfa4876c417e2a9935d58100d44d94bb41cd5bfb",
        "Debug: CI test found: pr: 21304, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/01f6c450b655a8ce233cb5feeaddb4ec8a5610f7",
        "Debug: CI test found: pr: 21305, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/547f97d5714b6e1efb8c64fd60b001c13c8880d0",
        "Debug: CI test found: pr: 21310, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/0078bbb232ce7dc6f6bf30bba8b8b26645f58b26",
        "Debug: CI test found: pr: 21312, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/df8a8240d33a04f66a0ffd756fac2d35b52274da",
        "Debug: merged PR without CI test: 21313",
        "Debug: CI test found: pr: 21314, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/a942a2784005212c34bba790fd7d4688d3bc098c",
        "Debug: CI test found: pr: 21316, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/3fec0e6cbfbfa17e018d88efbdf2e7a9231f364b",
        "Debug: CI test found: pr: 21318, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/db4f0788c51b7ec70369604a8a327e682c98405c",
        "Debug: CI test found: pr: 21320, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/875afa02fabe1dad5aa3d1e9bff89d493a369fd0",
        "Debug: CI test found: pr: 21321, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/a55277b88966d7449222c21a663b7dd9d3d330bb",
        "Debug: CI test found: pr: 21324, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/b9df4a2b208f586838ecf19648169406cf8c71e9",
        "Debug: merged PR without CI test: 21325",
        "Debug: CI test found: pr: 21327, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/44ddfb922f1434411763a9a0d92d4778f75054f8",
        "Debug: CI test found: pr: 21328, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/5e034d4d32ce2a0374c9613cf2660d146a35e830",
        "Debug: merged PR without CI test: 21329",
        "Debug: CI test found: pr: 21330, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/4df52c20f4ddc6249c114c4476b50ec9b640ec20",
        "Debug: CI test found: pr: 21331, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/5dd57a00d5c48e0553062e8eb39357c5f36df20b",
        "Debug: merged PR without CI test: 21332",
        "Debug: CI test found: pr: 21333, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/e0c311b1aa146852b992c1501d032a6908033c25",
        "Debug: CI test found: pr: 21334, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/55caae6a787ab328d7bc52adfd92aefecf789ec9",
        "Debug: CI test found: pr: 21335, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/f1d467af2589d129ccd6bba9ed7eaa0b77e568a2",
        "Debug: merged PR without CI test: 21337",
        "Debug: CI test found: pr: 21341, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/17f8d8f9b4bfaf0ea191409030c511cd713f64eb",
        "Debug: CI test found: pr: 21342, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/38ac3ab10aaad96ece4805ca29354e0006a51719",
        "Debug: CI test found: pr: 21347, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/4f8c9645dfaad98b544e0ba3fdc9258513d8669f",
        "Debug: CI test found: pr: 21349, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/502e2b4b9e1dd54a6ac29e7609c2d5e8aa220384"
```
With this patch:
```
        "Debug: CI test found: pr: 21299, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/472a52d22b9f505a4c284e7ad020ea1b3728466d",
        "Debug: CI test found: pr: 21300, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/cf35602cbc1bffcb5ab01153af153d0bf4e5a7e9",
        "Debug: CI test found: pr: 21301, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/13b7b8bd7362d198e123a67b8dfdc1c558df5b21",
        "Debug: CI test found: pr: 21302, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/dfa4876c417e2a9935d58100d44d94bb41cd5bfb",
        "Debug: CI test found: pr: 21304, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/01f6c450b655a8ce233cb5feeaddb4ec8a5610f7",
        "Debug: CI test found: pr: 21305, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/547f97d5714b6e1efb8c64fd60b001c13c8880d0",
        "Debug: CI test found: pr: 21310, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/0078bbb232ce7dc6f6bf30bba8b8b26645f58b26",
        "Debug: CI test found: pr: 21312, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/df8a8240d33a04f66a0ffd756fac2d35b52274da",
        "Debug: CI test found: pr: 21313, context: github-actions: https://api.github.com/repos/systemd/systemd/check-runs/4191612395",
        "Debug: CI test found: pr: 21314, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/a942a2784005212c34bba790fd7d4688d3bc098c",
        "Debug: CI test found: pr: 21316, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/3fec0e6cbfbfa17e018d88efbdf2e7a9231f364b",
        "Debug: CI test found: pr: 21318, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/db4f0788c51b7ec70369604a8a327e682c98405c",
        "Debug: CI test found: pr: 21320, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/875afa02fabe1dad5aa3d1e9bff89d493a369fd0",
        "Debug: CI test found: pr: 21321, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/a55277b88966d7449222c21a663b7dd9d3d330bb",
        "Debug: CI test found: pr: 21324, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/b9df4a2b208f586838ecf19648169406cf8c71e9",
        "Debug: CI test found: pr: 21325, context: github-actions: https://api.github.com/repos/systemd/systemd/check-runs/4191237494",
        "Debug: CI test found: pr: 21327, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/44ddfb922f1434411763a9a0d92d4778f75054f8",
        "Debug: CI test found: pr: 21328, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/5e034d4d32ce2a0374c9613cf2660d146a35e830",
        "Debug: CI test found: pr: 21329, context: github-actions: https://api.github.com/repos/systemd/systemd/check-runs/4192198481",
        "Debug: CI test found: pr: 21330, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/4df52c20f4ddc6249c114c4476b50ec9b640ec20",
        "Debug: CI test found: pr: 21331, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/5dd57a00d5c48e0553062e8eb39357c5f36df20b",
        "Debug: CI test found: pr: 21332, context: github-actions: https://api.github.com/repos/systemd/systemd/check-runs/4192365458",
        "Debug: CI test found: pr: 21333, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/e0c311b1aa146852b992c1501d032a6908033c25",
        "Debug: CI test found: pr: 21334, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/55caae6a787ab328d7bc52adfd92aefecf789ec9",
        "Debug: CI test found: pr: 21335, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/f1d467af2589d129ccd6bba9ed7eaa0b77e568a2",
        "Debug: CI test found: pr: 21337, context: github-actions: https://api.github.com/repos/systemd/systemd/check-runs/4197451714",
        "Debug: CI test found: pr: 21341, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/17f8d8f9b4bfaf0ea191409030c511cd713f64eb",
        "Debug: CI test found: pr: 21342, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/38ac3ab10aaad96ece4805ca29354e0006a51719",
        "Debug: CI test found: pr: 21347, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/4f8c9645dfaad98b544e0ba3fdc9258513d8669f",
        "Debug: CI test found: pr: 21349, context: ci/semaphoreci/pr: Debian autopkgtest (LXC): https://api.github.com/repos/systemd/systemd/statuses/502e2b4b9e1dd54a6ac29e7609c2d5e8aa220384"
```

Co-authored-by: Naveen <[email protected]>
  • Loading branch information
@evverx @naveensrinivasan
evverx and naveensrinivasan authored Nov 15, 2021
1 parent 16cd53d commit d490455
Showing 1 changed file with 17 additions and 30 deletions.
47 changes: 17 additions & 30 deletions checks/ci_tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,10 @@ import (
sce "github.com/ossf/scorecard/v3/errors"
)

// States for which CI system is in use.
type ciSystemState int

const (
// CheckCITests is the registered name for CITests.
CheckCITests = "CI-Tests"
success = "success"
unknown ciSystemState = iota
githubStatuses
githubCheckRuns
CheckCITests = "CI-Tests"
success = "success"
)

//nolint:gochecknoinits
Expand All @@ -48,7 +42,6 @@ func CITests(c *checker.CheckRequest) checker.CheckResult {
return checker.CreateRuntimeErrorResult(CheckCITests, e)
}

usedSystem := unknown
totalMerged := 0
totalTested := 0
for index := range prs {
Expand All @@ -61,30 +54,24 @@ func CITests(c *checker.CheckRequest) checker.CheckResult {
var foundCI bool

// Github Statuses.
if usedSystem != githubCheckRuns {
prSuccessStatus, err := prHasSuccessStatus(pr, c)
if err != nil {
return checker.CreateRuntimeErrorResult(CheckCITests, err)
}
if prSuccessStatus {
totalTested++
foundCI = true
usedSystem = githubStatuses
continue
}
prSuccessStatus, err := prHasSuccessStatus(pr, c)
if err != nil {
return checker.CreateRuntimeErrorResult(CheckCITests, err)
}
if prSuccessStatus {
totalTested++
foundCI = true
continue
}

// Github Check Runs.
if usedSystem != githubStatuses {
prCheckSuccessful, err := prHasSuccessfulCheck(pr, c)
if err != nil {
return checker.CreateRuntimeErrorResult(CheckCITests, err)
}
if prCheckSuccessful {
totalTested++
foundCI = true
usedSystem = githubCheckRuns
}
prCheckSuccessful, err := prHasSuccessfulCheck(pr, c)
if err != nil {
return checker.CreateRuntimeErrorResult(CheckCITests, err)
}
if prCheckSuccessful {
totalTested++
foundCI = true
}

if !foundCI {
Expand Down

0 comments on commit d490455

Please sign in to comment.