Update e2e tests (#1641)

Co-authored-by: Azeem Shaikh <[email protected]>
ossf · Feb 15, 2022 · de5224b · de5224b
1 parent 2b206dc
commit de5224b
Show file tree

Hide file tree

Showing 27 changed files with 37 additions and 153 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -72,7 +72,7 @@ jobs:
           max_attempts: 3
           retry_on: error
           timeout_minutes: 30
-          command: make ci-e2e
+          command: make e2e
 
       - name: codecov
         uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # 2.1.0

diff --git a/Makefile b/Makefile
@@ -10,7 +10,6 @@ PROTOC := $(shell which protoc)
 GORELEASER := goreleaser
 IMAGE_NAME = scorecard
 OUTPUT = output
-IGNORED_CI_TEST="E2E TEST:blob|E2E TEST:executable"
 PLATFORM="linux/amd64,linux/arm64,linux/386,linux/arm"
 LDFLAGS=$(shell ./scripts/version-ldflags)
 
@@ -264,23 +263,15 @@ unit-test: ## Runs unit test without e2e
 	# run the go tests and gen the file coverage-all used to do the integration with codecov
 	go test -race -covermode=atomic  -coverprofile=unit-coverage.out `go list ./... | grep -v e2e`
 
-e2e: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
-e2e: build-scorecard check-env | $(GINKGO)
-	# Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
-	$(GINKGO) --race --skip="E2E TEST:executable" -p -v -cover  -coverprofile=filename ./...
-
 $(GINKGO): install
 
-ci-e2e: ## Runs CI e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
-ci-e2e: build-scorecard check-env | $(GINKGO)
-	# Run CI e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
-	$(call ndef, GITHUB_AUTH_TOKEN)
-	@echo Ignoring these test for ci-e2e $(IGNORED_CI_TEST)
-	$(GINKGO) -p  -v -cover --skip=$(IGNORED_CI_TEST) -coverprofile=e2e-coverage.out ./e2e/...
-
-
 check-env:
 ifndef GITHUB_AUTH_TOKEN
 	$(error GITHUB_AUTH_TOKEN is undefined)
 endif
+
+e2e: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
+e2e: build-scorecard check-env | $(GINKGO)
+	# Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
+	$(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out ./...
 ###############################################################################
diff --git a/e2e/binary_artifacts_test.go b/e2e/binary_artifacts_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/cii_best_practices_test.go b/e2e/cii_best_practices_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"
@@ -27,7 +27,7 @@ import (
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
-var _ = Describe("E2E TEST:CIIBestPractices", func() {
+var _ = Describe("E2E TEST:"+checks.CheckCIIBestPractices, func() {
 	Context("E2E TEST:Validating use of CII Best Practices", func() {
 		It("Should return use of CII Best Practices", func() {
 			dl := scut.TestDetailLogger{}

diff --git a/e2e/code_review_test.go b/e2e/code_review_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"
@@ -29,7 +29,7 @@ import (
 
 // TODO: use dedicated repo that don't change.
 // TODO: need negative results.
-var _ = Describe("E2E TEST:CodeReview", func() {
+var _ = Describe("E2E TEST:"+checks.CheckCodeReview, func() {
 	Context("E2E TEST:Validating use of code reviews", func() {
 		It("Should return use of code reviews", func() {
 			dl := scut.TestDetailLogger{}

diff --git a/e2e/contributors_test.go b/e2e/contributors_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/dangerous_workflow_test.go b/e2e/dangerous_workflow_test.go
@@ -16,7 +16,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/dependency_update_tool_test.go b/e2e/dependency_update_tool_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/e2e_suite_test.go b/e2e/e2e_suite_test.go
@@ -15,10 +15,9 @@
 package e2e
 
 import (
-	"os"
 	"testing"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/log"
@@ -31,18 +30,3 @@ func TestE2e(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "E2e Suite")
 }
-
-var _ = BeforeSuite(func() {
-	// making sure the GITHUB_AUTH_TOKEN is set prior to running e2e tests
-	token, contains := os.LookupEnv("GITHUB_AUTH_TOKEN")
-
-	Expect(contains).ShouldNot(BeFalse(),
-		"GITHUB_AUTH_TOKEN env variable is not set.The GITHUB_AUTH_TOKEN env variable has to be set to run e2e test.")
-	Expect(len(token)).ShouldNot(BeZero(), "Length of the GITHUB_AUTH_TOKEN env variable is zero.")
-
-	l := log.NewLogger(log.InfoLevel)
-	logger = l
-})
-
-var _ = AfterSuite(func() {
-})
diff --git a/e2e/executable_test.go b/e2e/executable_test.go
diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/license_test.go b/e2e/license_test.go
@@ -16,7 +16,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/maintained_test.go b/e2e/maintained_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/permissions_test.go b/e2e/permissions_test.go
@@ -16,7 +16,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/pinned_dependencies_test.go b/e2e/pinned_dependencies_test.go
@@ -16,7 +16,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/sast_test.go b/e2e/sast_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/security_policy_test.go b/e2e/security_policy_test.go
@@ -16,7 +16,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"
@@ -26,7 +26,7 @@ import (
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
-var _ = Describe("E2E TEST:SecurityPolicy", func() {
+var _ = Describe("E2E TEST:"+checks.CheckSecurityPolicy, func() {
 	Context("E2E TEST:Validating security policy", func() {
 		It("Should return valid security policy", func() {
 			dl := scut.TestDetailLogger{}

diff --git a/e2e/signedreleases_test.go b/e2e/signedreleases_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"

diff --git a/e2e/vulnerabilities_test.go b/e2e/vulnerabilities_test.go
@@ -17,7 +17,7 @@ package e2e
 import (
 	"context"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/ossf/scorecard/v4/checker"
@@ -27,7 +27,7 @@ import (
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
-var _ = Describe("E2E TEST:Vulnerabilities", func() {
+var _ = Describe("E2E TEST:"+checks.CheckVulnerabilities, func() {
 	Context("E2E TEST:Validating vulnerabilities status", func() {
 		It("Should return that there are no vulnerabilities", func() {
 			repo, err := githubrepo.MakeGithubRepo("ossf/scorecard")