From 2cef957e2165626ae2279b3513761e59acb33c0d Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Thu, 18 Feb 2021 11:11:54 +0530 Subject: [PATCH] build pipeline changes and more tests --- .github/workflows/gobuild.yml | 3 + Makefile | 2 +- scripts/generate-coverage.sh | 2 +- scripts/run-e2e.sh | 3 +- test/e2e/init/config/invalid_path.toml | 2 + test/e2e/init/config/valid_config.toml | 3 + test/e2e/init/golden/init_help.txt | 2 +- test/e2e/init/ignore.go | 3 + test/e2e/init/init_test.go | 107 +++++++++++++++++++------ test/helper/helper.go | 18 ++--- tools/tools.go | 13 +++ 11 files changed, 119 insertions(+), 39 deletions(-) create mode 100644 test/e2e/init/config/invalid_path.toml create mode 100644 test/e2e/init/config/valid_config.toml create mode 100644 test/e2e/init/ignore.go create mode 100644 tools/tools.go diff --git a/.github/workflows/gobuild.yml b/.github/workflows/gobuild.yml index d950db96f..86e8197fe 100644 --- a/.github/workflows/gobuild.yml +++ b/.github/workflows/gobuild.yml @@ -30,6 +30,9 @@ jobs: - name: Run unit tests run: make unit-tests + - name: Run e2e tests + run: make e2e-tests + - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/Makefile b/Makefile index 3b82eaa0c..a9b621d28 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ cicd: validate build test docker-build # run all unit and integration tests -test: unit-tests +test: unit-tests e2e-tests # run all validation tests diff --git a/scripts/generate-coverage.sh b/scripts/generate-coverage.sh index 90978c878..ba3d1e1ee 100755 --- a/scripts/generate-coverage.sh +++ b/scripts/generate-coverage.sh @@ -4,5 +4,5 @@ set -o errexit set -o nounset set -o pipefail -go test -v -coverpkg=./... -coverprofile=coverage.out ./... +go test -v -coverpkg=./pkg/... -coverprofile=coverage.out ./pkg/... go tool cover -func coverage.out diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh index 46eb3020b..dc7b67d1c 100755 --- a/scripts/run-e2e.sh +++ b/scripts/run-e2e.sh @@ -6,4 +6,5 @@ set -o pipefail export TERRASCAN_BIN_PATH=${PWD}/bin/terrascan -go test -v ./test/... \ No newline at end of file +go test -v -coverpkg=./test/... -coverprofile=coverage.out ./test/... +go tool cover -func coverage.out \ No newline at end of file diff --git a/test/e2e/init/config/invalid_path.toml b/test/e2e/init/config/invalid_path.toml new file mode 100644 index 000000000..bcdcceaf4 --- /dev/null +++ b/test/e2e/init/config/invalid_path.toml @@ -0,0 +1,2 @@ +[policy] +path = "invalid/path" \ No newline at end of file diff --git a/test/e2e/init/config/valid_config.toml b/test/e2e/init/config/valid_config.toml new file mode 100644 index 000000000..f3f62657f --- /dev/null +++ b/test/e2e/init/config/valid_config.toml @@ -0,0 +1,3 @@ +[policy] +repo_url = "https://github.com/accurics/KaiMonkey.git" +branch = "master" \ No newline at end of file diff --git a/test/e2e/init/golden/init_help.txt b/test/e2e/init/golden/init_help.txt index 2195be35f..0a2d2a485 100644 --- a/test/e2e/init/golden/init_help.txt +++ b/test/e2e/init/golden/init_help.txt @@ -12,4 +12,4 @@ Global Flags: -c, --config-path string config file path -l, --log-level string log level (debug, info, warn, error, panic, fatal) (default "info") -x, --log-type string log output type (console, json) (default "console") - -o, --output string output type (human, json, yaml, xml) (default "human") + -o, --output string output type (human, json, yaml, xml, junit-xml) (default "human") diff --git a/test/e2e/init/ignore.go b/test/e2e/init/ignore.go new file mode 100644 index 000000000..235dab9fb --- /dev/null +++ b/test/e2e/init/ignore.go @@ -0,0 +1,3 @@ +package init + +// This file is added to ignore 'no non-test Go files' issue diff --git a/test/e2e/init/init_test.go b/test/e2e/init/init_test.go index added5313..d19b00891 100644 --- a/test/e2e/init/init_test.go +++ b/test/e2e/init/init_test.go @@ -4,6 +4,7 @@ import ( "io" "os" "path/filepath" + "time" "github.com/accurics/terrascan/test/helper" . "github.com/onsi/ginkgo" @@ -18,10 +19,12 @@ const ( ) var ( + initCommand string = "init" defaultPolicyRepoPath string = os.Getenv("HOME") + "/.terrascan" terrascanGitURL string = "https://github.com/accurics/terrascan.git" terrascanDefaultBranch string = "master" terrascanConfigEnvName string = "TERRASCAN_CONFIG" + kaiMoneyGitURL string = "https://github.com/accurics/KaiMonkey.git" ) var _ = Describe("Init", func() { @@ -46,10 +49,9 @@ var _ = Describe("Init", func() { }) Describe("terrascan init is run", func() { - When("terrascan init is run without any flags", func() { + When("without any flags", func() { It("should download policies and exit with status code 0", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") - Eventually(session, initCommandTimeout).Should(gexec.Exit(0)) + session = runInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) Expect(outWriter).Should(gbytes.Say("")) }) @@ -61,19 +63,10 @@ var _ = Describe("Init", func() { var repo *git.Repository var err error It("should be a valid git repo", func() { - repo, err = git.PlainOpen(defaultPolicyRepoPath) - Expect(err).NotTo(HaveOccurred()) - Expect(repo).NotTo(BeNil()) + repo = openGitRepo(defaultPolicyRepoPath) }) It("should be terrascan git repo", func() { - remote, err := repo.Remote("origin") - Expect(err).NotTo(HaveOccurred()) - Expect(remote).NotTo(BeNil()) - remoteConfig := remote.Config() - Expect(remoteConfig).NotTo(BeNil()) - err = remoteConfig.Validate() - Expect(err).NotTo(HaveOccurred()) - Expect(remoteConfig.URLs[0]).To(BeEquivalentTo(terrascanGitURL)) + validateGitRepo(repo, terrascanGitURL) }) It("master branch should be present", func() { _, err = repo.Branch(terrascanDefaultBranch) @@ -84,14 +77,14 @@ var _ = Describe("Init", func() { When("terrascan init is run with -h flag", func() { It("should print help", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init", "-h") + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, initCommand, "-h") goldenFileAbsPath, err := filepath.Abs("golden/init_help.txt") Expect(err).NotTo(HaveOccurred()) helper.CompareActualWithGolden(session, goldenFileAbsPath, true) }) It("should exit with status code 0", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init", "-h") + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, initCommand, "-h") Eventually(session).Should(gexec.Exit(0)) }) }) @@ -128,9 +121,8 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") - Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) - helper.ContainsErrorSubString(session, `failed to download policies. error: 'Get "https://repository/url/info/refs?service=git-upload-pack": dial tcp: lookup repository on 8.8.8.8:53: no such host'`) + session = runInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) + helper.ContainsErrorSubString(session, `failed to download policies. error: 'Get "https://repository/url/info/refs?service=git-upload-pack": dial tcp:`) }) }) When("the config file has invalid branch name", func() { @@ -141,9 +133,8 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") - Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) - helper.ContainsErrorSubString(session, `failed to checkout branch 'invalid-branch'. error: 'reference not found'`) + session = runInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) + helper.ContainsErrorSubString(session, `failed to initialize terrascan. error : failed to checkout git branch 'invalid-branch'. error: 'reference not found'`) }) }) When("the config file has invalid rego subdir", func() { @@ -154,8 +145,7 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") - Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) + session = runInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) helper.ContainsErrorSubString(session, "invalid/path: no such file or directory") }) }) @@ -166,9 +156,74 @@ var _ = Describe("Init", func() { JustAfterEach(func() { os.Setenv(terrascanConfigEnvName, "") }) - It("should error out and exit with status code 1", func() { - Skip("Skipping invalid path test until discussion with team") + It("should should download policies and exit with status code 0", func() { + runInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + }) + }) + Context("the config file has valid data", func() { + When("config file has different git repo and branch", func() { + JustBeforeEach(func() { + os.Setenv(terrascanConfigEnvName, "config/valid_config.toml") + }) + JustAfterEach(func() { + os.Setenv(terrascanConfigEnvName, "") + }) + It("init should download the repo provided in the config file", func() { + runInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + }) + Context("Kai Monkey git repo is downloaded", func() { + It("should validate Kai Monkey repo in the policy path", func() { + repo := openGitRepo(defaultPolicyRepoPath) + validateGitRepo(repo, kaiMoneyGitURL) + }) + }) + }) + }) + }) + + Describe("terrascan init is run multiple times", func() { + Context("init clones the git repo to a temp dir, deletes policy path and renames tempdir to policy path", func() { + Context("running init the first time", func() { + var modifiedTime time.Time + It("should download policies at the default policy path", func() { + runInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + fi, err := os.Stat(defaultPolicyRepoPath) + Expect(err).ToNot(HaveOccurred()) + modifiedTime = fi.ModTime() + }) + Context("running init the second time", func() { + It("should download policies again at the default policy path", func() { + runInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + fi, err := os.Stat(defaultPolicyRepoPath) + Expect(err).ToNot(HaveOccurred()) + Expect(fi.ModTime()).To(BeTemporally(">", modifiedTime)) + }) + }) }) }) }) }) + +func runInitCommand(terrascanBinaryPath string, outWriter, errWriter io.Writer, exitCode int) *gexec.Session { + session := helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, initCommand) + Eventually(session, initCommandTimeout).Should(gexec.Exit(exitCode)) + return session +} + +func openGitRepo(repoPath string) *git.Repository { + repo, err := git.PlainOpen(repoPath) + Expect(err).NotTo(HaveOccurred()) + Expect(repo).NotTo(BeNil()) + return repo +} + +func validateGitRepo(repo *git.Repository, gitURL string) { + remote, err := repo.Remote("origin") + Expect(err).NotTo(HaveOccurred()) + Expect(remote).NotTo(BeNil()) + remoteConfig := remote.Config() + Expect(remoteConfig).NotTo(BeNil()) + err = remoteConfig.Validate() + Expect(err).NotTo(HaveOccurred()) + Expect(remoteConfig.URLs[0]).To(BeEquivalentTo(gitURL)) +} diff --git a/test/helper/helper.go b/test/helper/helper.go index 9eae13800..88b77c0ae 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -6,33 +6,33 @@ import ( "os" "os/exec" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" ) // CompareActualWithGolden compares func CompareActualWithGolden(session *gexec.Session, goldenFileAbsPath string, isStdOut bool) { fileData, err := ioutil.ReadFile(goldenFileAbsPath) - Expect(err).NotTo(HaveOccurred()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) if isStdOut { - Expect(string(session.Wait().Out.Contents())).Should(BeIdenticalTo(string(fileData))) + gomega.Expect(string(session.Wait().Out.Contents())).Should(gomega.BeIdenticalTo(string(fileData))) } else { - Expect(string(session.Wait().Err.Contents())).Should(BeIdenticalTo(string(fileData))) + gomega.Expect(string(session.Wait().Err.Contents())).Should(gomega.BeIdenticalTo(string(fileData))) } } // ContainsErrorSubString will assert if error string is part of error output func ContainsErrorSubString(session *gexec.Session, errSubString string) { - Expect(string(session.Wait().Err.Contents())).Should(ContainSubstring(errSubString)) + gomega.Expect(string(session.Wait().Err.Contents())).Should(gomega.ContainSubstring(errSubString)) } // GetTerrascanBinaryPath returns the terrascan binary path func GetTerrascanBinaryPath() string { terrascanBinaryPath := os.Getenv("TERRASCAN_BIN_PATH") - Describe("terrascan binary path should be set for executing tests", func() { + ginkgo.Describe("terrascan binary path should be set for executing tests", func() { if terrascanBinaryPath == "" { - Fail("ensure that TERRASCAN_BIN_PATH is set") + ginkgo.Fail("ensure that TERRASCAN_BIN_PATH is set") } }) return terrascanBinaryPath @@ -42,6 +42,6 @@ func GetTerrascanBinaryPath() string { func RunCommand(path string, outWriter, errWriter io.Writer, args ...string) *gexec.Session { cmd := exec.Command(path, args...) session, err := gexec.Start(cmd, outWriter, errWriter) - Expect(err).NotTo(HaveOccurred()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) return session } diff --git a/tools/tools.go b/tools/tools.go new file mode 100644 index 000000000..8cf1325b6 --- /dev/null +++ b/tools/tools.go @@ -0,0 +1,13 @@ +// +build tools + +package tools + +import ( + // used only for testing + _ "github.com/onsi/ginkgo/ginkgo" + // used only for testing + _ "github.com/onsi/gomega" +) + +// This file imports packages that are used when running go generate, or used +// during the development process but not otherwise depended on by built code.