🌱 Fixes the broken e2e

Fixes for broken e2e
ossf · Oct 27, 2021 · aa634bd · aa634bd
1 parent fd238d0
commit aa634bd
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 105 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -86,9 +86,8 @@ jobs:
         env:
           GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
         run: |
-              echo "Skipping ci-e2e. Will be re-enabled soon."
-              #go env -w GOFLAGS=-mod=mod
-              #make ci-e2e
+              go env -w GOFLAGS=-mod=mod
+              make ci-e2e
 
       - name: install tools
         run: make install

diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go
@@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() {
 	Context("E2E TEST:Validating branch protection", func() {
 		It("Should fail to return branch protection on other repositories", func() {
 			dl := scut.TestDetailLogger{}
-			repo, err := githubrepo.MakeGithubRepo("apache/airflow")
+			repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e")
 			Expect(err).Should(BeNil())
 			repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
 			err = repoClient.InitRepo(repo)
@@ -43,48 +43,18 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() {
 			}
 			expected := scut.TestReturn{
 				Error:         nil,
-				Score:         1,
-				NumberOfWarn:  3,
-				NumberOfInfo:  0,
+				Score:         9,
+				NumberOfWarn:  1,
+				NumberOfInfo:  8,
 				NumberOfDebug: 0,
 			}
 			result := checks.BranchProtection(&req)
 			// UPGRADEv2: to remove.
 			// Old version.
-			Expect(result.Error).ShouldNot(BeNil())
-			Expect(result.Pass).Should(BeFalse())
+			Expect(result.Error).Should(BeNil())
+			Expect(result.Pass).Should(BeTrue())
 			// New version.
-			Expect(scut.ValidateTestReturn(nil, "branch protection not accessible", &expected, &result, &dl)).Should(BeTrue())
-		})
-		Context("E2E TEST:Validating branch protection", func() {
-			It("Should fail to return branch protection on other repositories", func() {
-				dl := scut.TestDetailLogger{}
-				repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e")
-				Expect(err).Should(BeNil())
-				repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
-				err = repoClient.InitRepo(repo)
-				Expect(err).Should(BeNil())
-				req := checker.CheckRequest{
-					Ctx:        context.Background(),
-					RepoClient: repoClient,
-					Repo:       repo,
-					Dlogger:    &dl,
-				}
-				expected := scut.TestReturn{
-					Error:         nil,
-					Score:         9,
-					NumberOfWarn:  1,
-					NumberOfInfo:  8,
-					NumberOfDebug: 0,
-				}
-				result := checks.BranchProtection(&req)
-				// UPGRADEv2: to remove.
-				// Old version.
-				Expect(result.Error).Should(BeNil())
-				Expect(result.Pass).Should(BeTrue())
-				// New version.
-				Expect(scut.ValidateTestReturn(nil, "branch protection accessible", &expected, &result, &dl)).Should(BeTrue())
-			})
+			Expect(scut.ValidateTestReturn(nil, "branch protection accessible", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})
 })
diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go
@@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
 	Context("E2E TEST:Validating use of CI tests", func() {
 		It("Should return use of CI tests", func() {
 			dl := scut.TestDetailLogger{}
-			repo, err := githubrepo.MakeGithubRepo("apache/airflow")
+			repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow")
 			Expect(err).Should(BeNil())
 			repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
 			err = repoClient.InitRepo(repo)
@@ -43,17 +43,12 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
 			}
 			expected := scut.TestReturn{
 				Error:         nil,
-				Score:         checker.MaxResultScore,
+				Score:         checker.InconclusiveResultScore,
 				NumberOfWarn:  0,
 				NumberOfInfo:  0,
-				NumberOfDebug: 30,
+				NumberOfDebug: 0,
 			}
 			result := checks.CITests(&req)
-			// UPGRADEv2: to remove.
-			// Old version.
-			Expect(result.Error).Should(BeNil())
-			Expect(result.Pass).Should(BeTrue())
-			// New version.
 			Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})

diff --git a/e2e/code_review_test.go b/e2e/code_review_test.go
@@ -32,7 +32,7 @@ var _ = Describe("E2E TEST:CodeReview", func() {
 	Context("E2E TEST:Validating use of code reviews", func() {
 		It("Should return use of code reviews", func() {
 			dl := scut.TestDetailLogger{}
-			repo, err := githubrepo.MakeGithubRepo("apache/airflow")
+			repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow")
 			Expect(err).Should(BeNil())
 			repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
 			err = repoClient.InitRepo(repo)
@@ -46,17 +46,12 @@ var _ = Describe("E2E TEST:CodeReview", func() {
 			}
 			expected := scut.TestReturn{
 				Error:         nil,
-				Score:         checker.MaxResultScore,
+				Score:         checker.MinResultScore,
 				NumberOfWarn:  0,
-				NumberOfInfo:  0,
+				NumberOfInfo:  3,
 				NumberOfDebug: 0,
 			}
 			result := checks.DoesCodeReview(&req)
-			// UPGRADEv2: to remove.
-			// Old version.
-			Expect(result.Error).Should(BeNil())
-			Expect(result.Pass).Should(BeTrue())
-			// New version.
 			Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})

diff --git a/e2e/contributors_test.go b/e2e/contributors_test.go
@@ -56,33 +56,5 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() {
 			// New version.
 			Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue())
 		})
-		It("Should return valid project contributors", func() {
-			dl := scut.TestDetailLogger{}
-			repo, err := githubrepo.MakeGithubRepo("apache/airflow")
-			Expect(err).Should(BeNil())
-			repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
-			err = repoClient.InitRepo(repo)
-			Expect(err).Should(BeNil())
-			checkRequest := checker.CheckRequest{
-				Ctx:        context.Background(),
-				RepoClient: repoClient,
-				Repo:       repo,
-				Dlogger:    &dl,
-			}
-			expected := scut.TestReturn{
-				Error:         nil,
-				Score:         checker.MaxResultScore,
-				NumberOfWarn:  0,
-				NumberOfInfo:  1,
-				NumberOfDebug: 0,
-			}
-			result := checks.Contributors(&checkRequest)
-			// UPGRADEv2: to remove.
-			// Old version.
-			Expect(result.Error).Should(BeNil())
-			Expect(result.Pass).Should(BeTrue())
-			// New version.
-			Expect(scut.ValidateTestReturn(nil, "several contributors", &expected, &result, &dl)).Should(BeTrue())
-		})
 	})
 })
diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go
@@ -43,18 +43,12 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() {
 			}
 			expected := scut.TestReturn{
 				Error:         nil,
-				Score:         checker.MaxResultScore,
-				NumberOfWarn:  0,
+				Score:         checker.InconclusiveResultScore,
+				NumberOfWarn:  1,
 				NumberOfInfo:  2,
 				NumberOfDebug: 2,
 			}
 			result := checks.Packaging(&req)
-
-			// UPGRADEv2: to remove.
-			// Old version.
-			Expect(result.Error).Should(BeNil())
-			Expect(result.Pass).Should(BeTrue())
-			// New version.
 			Expect(scut.ValidateTestReturn(nil, "use packaging", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})

diff --git a/e2e/pinned_dependencies_test.go b/e2e/pinned_dependencies_test.go
@@ -45,17 +45,12 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() {
 			}
 			expected := scut.TestReturn{
 				Error:         nil,
-				Score:         checker.MinResultScore,
-				NumberOfWarn:  154,
-				NumberOfInfo:  0,
+				Score:         3,
+				NumberOfWarn:  150,
+				NumberOfInfo:  2,
 				NumberOfDebug: 0,
 			}
 			result := checks.PinnedDependencies(&req)
-			// UPGRADEv2: to remove.
-			// Old version.
-			Expect(result.Error).Should(BeNil())
-			Expect(result.Pass).Should(BeFalse())
-			// New version.
 			Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue())
 		})
 	})

diff --git a/e2e/sast_test.go b/e2e/sast_test.go
@@ -30,7 +30,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() {
 	Context("E2E TEST:Validating use of SAST tools", func() {
 		It("Should return use of SAST tools", func() {
 			dl := scut.TestDetailLogger{}
-			repo, err := githubrepo.MakeGithubRepo("apache/airflow")
+			repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow")
 			Expect(err).Should(BeNil())
 			repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
 			err = repoClient.InitRepo(repo)
@@ -43,10 +43,10 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() {
 			}
 			expected := scut.TestReturn{
 				Error:         nil,
-				Score:         7,
-				NumberOfWarn:  1,
-				NumberOfInfo:  1,
-				NumberOfDebug: 1,
+				Score:         0,
+				NumberOfWarn:  2,
+				NumberOfInfo:  0,
+				NumberOfDebug: 0,
 			}
 			result := checks.SAST(&req)
 			// UPGRADEv2: to remove.