From 456cb318e01c200d745d8390014569cbb1de164c Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Tue, 16 Feb 2021 18:17:04 +0530 Subject: [PATCH 1/2] e2e init tests --- Makefile | 4 + go.mod | 2 + go.sum | 12 ++ scripts/run-e2e.sh | 9 + test/e2e/init/config/invalid_branch.toml | 2 + test/e2e/init/config/invalid_rego_subdir.toml | 2 + test/e2e/init/config/invalid_repo.toml | 2 + test/e2e/init/golden/init_help.txt | 15 ++ test/e2e/init/golden/init_typo_help.txt | 6 + test/e2e/init/init_suite_test.go | 13 ++ test/e2e/init/init_test.go | 174 ++++++++++++++++++ test/helper/helper.go | 47 +++++ 12 files changed, 288 insertions(+) create mode 100755 scripts/run-e2e.sh create mode 100644 test/e2e/init/config/invalid_branch.toml create mode 100644 test/e2e/init/config/invalid_rego_subdir.toml create mode 100644 test/e2e/init/config/invalid_repo.toml create mode 100644 test/e2e/init/golden/init_help.txt create mode 100644 test/e2e/init/golden/init_typo_help.txt create mode 100644 test/e2e/init/init_suite_test.go create mode 100644 test/e2e/init/init_test.go create mode 100644 test/helper/helper.go diff --git a/Makefile b/Makefile index be3133306..3b82eaa0c 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ help: @echo "staticcheck\n\trun static code analysis" @echo "test\n\texecute unit and integration tests" @echo "unit-tests\n\texecute unit tests" + @echo "e2e-tests\n\texecute e2e tests" @echo "validate\n\trun all validations" # build terrascan binary @@ -82,6 +83,9 @@ staticcheck: unit-tests: ./scripts/generate-coverage.sh +# run e2e tests +e2e-tests: build + ./scripts/run-e2e.sh # build terrascan docker image docker-build: diff --git a/go.mod b/go.mod index 6b2394dd3..9b1248f23 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,8 @@ require ( github.com/hashicorp/terraform v0.14.4 github.com/iancoleman/strcase v0.1.3 github.com/mattn/go-isatty v0.0.12 + github.com/onsi/ginkgo v1.12.1 + github.com/onsi/gomega v1.10.5 github.com/open-policy-agent/opa v0.22.0 github.com/pelletier/go-toml v1.8.1 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 315261adb..605fbf557 100644 --- a/go.sum +++ b/go.sum @@ -761,6 +761,8 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -773,10 +775,15 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= +github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-policy-agent/opa v0.22.0 h1:KZvn0uMQIorBIwYk8Vc89dp8No9FIEF8eFl0sc1r/1U= github.com/open-policy-agent/opa v0.22.0/go.mod h1:rrwxoT/b011T0cyj+gg2VvxqTtn6N3gp/jzmr3fjW44= @@ -913,6 +920,7 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -1151,6 +1159,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1196,12 +1206,14 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh new file mode 100755 index 000000000..46eb3020b --- /dev/null +++ b/scripts/run-e2e.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +export TERRASCAN_BIN_PATH=${PWD}/bin/terrascan + +go test -v ./test/... \ No newline at end of file diff --git a/test/e2e/init/config/invalid_branch.toml b/test/e2e/init/config/invalid_branch.toml new file mode 100644 index 000000000..3b85c5075 --- /dev/null +++ b/test/e2e/init/config/invalid_branch.toml @@ -0,0 +1,2 @@ +[policy] +branch = "invalid-branch" \ No newline at end of file diff --git a/test/e2e/init/config/invalid_rego_subdir.toml b/test/e2e/init/config/invalid_rego_subdir.toml new file mode 100644 index 000000000..2b1cb5524 --- /dev/null +++ b/test/e2e/init/config/invalid_rego_subdir.toml @@ -0,0 +1,2 @@ +[policy] +rego_subdir = "invalid/path" \ No newline at end of file diff --git a/test/e2e/init/config/invalid_repo.toml b/test/e2e/init/config/invalid_repo.toml new file mode 100644 index 000000000..7477ad2c7 --- /dev/null +++ b/test/e2e/init/config/invalid_repo.toml @@ -0,0 +1,2 @@ +[policy] +repo_url = "https://repository/url" \ No newline at end of file diff --git a/test/e2e/init/golden/init_help.txt b/test/e2e/init/golden/init_help.txt new file mode 100644 index 000000000..2195be35f --- /dev/null +++ b/test/e2e/init/golden/init_help.txt @@ -0,0 +1,15 @@ +Terrascan + +Initializes Terrascan and clones policies from the Terrascan GitHub repository. + +Usage: + terrascan init [flags] + +Flags: + -h, --help help for init + +Global Flags: + -c, --config-path string config file path + -l, --log-level string log level (debug, info, warn, error, panic, fatal) (default "info") + -x, --log-type string log output type (console, json) (default "console") + -o, --output string output type (human, json, yaml, xml) (default "human") diff --git a/test/e2e/init/golden/init_typo_help.txt b/test/e2e/init/golden/init_typo_help.txt new file mode 100644 index 000000000..dbc393e84 --- /dev/null +++ b/test/e2e/init/golden/init_typo_help.txt @@ -0,0 +1,6 @@ +Error: unknown command "inti" for "terrascan" + +Did you mean this? + init + +Run 'terrascan --help' for usage. diff --git a/test/e2e/init/init_suite_test.go b/test/e2e/init/init_suite_test.go new file mode 100644 index 000000000..8fd8b634a --- /dev/null +++ b/test/e2e/init/init_suite_test.go @@ -0,0 +1,13 @@ +package init_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestInit(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Init Suite") +} diff --git a/test/e2e/init/init_test.go b/test/e2e/init/init_test.go new file mode 100644 index 000000000..added5313 --- /dev/null +++ b/test/e2e/init/init_test.go @@ -0,0 +1,174 @@ +package init_test + +import ( + "io" + "os" + "path/filepath" + + "github.com/accurics/terrascan/test/helper" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" + "gopkg.in/src-d/go-git.v4" +) + +const ( + initCommandTimeout = 60 +) + +var ( + defaultPolicyRepoPath string = os.Getenv("HOME") + "/.terrascan" + terrascanGitURL string = "https://github.com/accurics/terrascan.git" + terrascanDefaultBranch string = "master" + terrascanConfigEnvName string = "TERRASCAN_CONFIG" +) + +var _ = Describe("Init", func() { + var session *gexec.Session + var terrascanBinaryPath string + + var outWriter, errWriter io.Writer + + BeforeSuite(func() { + os.RemoveAll(defaultPolicyRepoPath) + terrascanBinaryPath = helper.GetTerrascanBinaryPath() + }) + + BeforeEach(func() { + outWriter = gbytes.NewBuffer() + errWriter = gbytes.NewBuffer() + }) + + AfterEach(func() { + outWriter = nil + errWriter = nil + }) + + Describe("terrascan init is run", func() { + When("terrascan init is run without any flags", func() { + It("should download policies and exit with status code 0", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") + Eventually(session, initCommandTimeout).Should(gexec.Exit(0)) + Expect(outWriter).Should(gbytes.Say("")) + }) + + It("should download policies at TERRASCAN's base location", func() { + BeADirectory().Match(defaultPolicyRepoPath) + }) + + Context("git repo should be validated", func() { + var repo *git.Repository + var err error + It("should be a valid git repo", func() { + repo, err = git.PlainOpen(defaultPolicyRepoPath) + Expect(err).NotTo(HaveOccurred()) + Expect(repo).NotTo(BeNil()) + }) + It("should be terrascan git repo", func() { + remote, err := repo.Remote("origin") + Expect(err).NotTo(HaveOccurred()) + Expect(remote).NotTo(BeNil()) + remoteConfig := remote.Config() + Expect(remoteConfig).NotTo(BeNil()) + err = remoteConfig.Validate() + Expect(err).NotTo(HaveOccurred()) + Expect(remoteConfig.URLs[0]).To(BeEquivalentTo(terrascanGitURL)) + }) + It("master branch should be present", func() { + _, err = repo.Branch(terrascanDefaultBranch) + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) + + When("terrascan init is run with -h flag", func() { + It("should print help", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init", "-h") + goldenFileAbsPath, err := filepath.Abs("golden/init_help.txt") + Expect(err).NotTo(HaveOccurred()) + helper.CompareActualWithGolden(session, goldenFileAbsPath, true) + }) + + It("should exit with status code 0", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init", "-h") + Eventually(session).Should(gexec.Exit(0)) + }) + }) + + When("terrascan init command has typo. eg: inti", func() { + It("should print command suggestion", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "inti") + goldenFileAbsPath, err := filepath.Abs("golden/init_typo_help.txt") + Expect(err).NotTo(HaveOccurred()) + helper.CompareActualWithGolden(session, goldenFileAbsPath, false) + }) + + It("should exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "inti") + Eventually(session, 5).Should(gexec.Exit(1)) + }) + }) + + When("terrascan init is run with -c flag", func() { + Context("config file has valid policy config data", func() { + It("should download policies as per the policy config in the config file", func() { + Skip("skipping this test due to https://github.com/accurics/terrascan/issues/550, should be implemented when fixed") + }) + }) + }) + }) + + Describe("terrascan init is run when TERRASCAN_CONFIG is set", func() { + When("the config file has invalid repo url", func() { + JustBeforeEach(func() { + os.Setenv(terrascanConfigEnvName, "config/invalid_repo.toml") + }) + JustAfterEach(func() { + os.Setenv(terrascanConfigEnvName, "") + }) + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") + Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) + helper.ContainsErrorSubString(session, `failed to download policies. error: 'Get "https://repository/url/info/refs?service=git-upload-pack": dial tcp: lookup repository on 8.8.8.8:53: no such host'`) + }) + }) + When("the config file has invalid branch name", func() { + JustBeforeEach(func() { + os.Setenv(terrascanConfigEnvName, "config/invalid_branch.toml") + }) + JustAfterEach(func() { + os.Setenv(terrascanConfigEnvName, "") + }) + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") + Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) + helper.ContainsErrorSubString(session, `failed to checkout branch 'invalid-branch'. error: 'reference not found'`) + }) + }) + When("the config file has invalid rego subdir", func() { + JustBeforeEach(func() { + os.Setenv(terrascanConfigEnvName, "config/invalid_rego_subdir.toml") + }) + JustAfterEach(func() { + os.Setenv(terrascanConfigEnvName, "") + }) + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") + Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) + helper.ContainsErrorSubString(session, "invalid/path: no such file or directory") + }) + }) + When("the config file has invalid path", func() { + JustBeforeEach(func() { + os.Setenv(terrascanConfigEnvName, "config/invalid_path.toml") + }) + JustAfterEach(func() { + os.Setenv(terrascanConfigEnvName, "") + }) + It("should error out and exit with status code 1", func() { + Skip("Skipping invalid path test until discussion with team") + }) + }) + }) +}) diff --git a/test/helper/helper.go b/test/helper/helper.go new file mode 100644 index 000000000..9eae13800 --- /dev/null +++ b/test/helper/helper.go @@ -0,0 +1,47 @@ +package helper + +import ( + "io" + "io/ioutil" + "os" + "os/exec" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" +) + +// CompareActualWithGolden compares +func CompareActualWithGolden(session *gexec.Session, goldenFileAbsPath string, isStdOut bool) { + fileData, err := ioutil.ReadFile(goldenFileAbsPath) + Expect(err).NotTo(HaveOccurred()) + if isStdOut { + Expect(string(session.Wait().Out.Contents())).Should(BeIdenticalTo(string(fileData))) + } else { + Expect(string(session.Wait().Err.Contents())).Should(BeIdenticalTo(string(fileData))) + } +} + +// ContainsErrorSubString will assert if error string is part of error output +func ContainsErrorSubString(session *gexec.Session, errSubString string) { + Expect(string(session.Wait().Err.Contents())).Should(ContainSubstring(errSubString)) +} + +// GetTerrascanBinaryPath returns the terrascan binary path +func GetTerrascanBinaryPath() string { + terrascanBinaryPath := os.Getenv("TERRASCAN_BIN_PATH") + Describe("terrascan binary path should be set for executing tests", func() { + if terrascanBinaryPath == "" { + Fail("ensure that TERRASCAN_BIN_PATH is set") + } + }) + return terrascanBinaryPath +} + +// RunCommand will return session +func RunCommand(path string, outWriter, errWriter io.Writer, args ...string) *gexec.Session { + cmd := exec.Command(path, args...) + session, err := gexec.Start(cmd, outWriter, errWriter) + Expect(err).NotTo(HaveOccurred()) + return session +} From f906aed373fa542b62c82a5ced2c2763b65218d3 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Thu, 18 Feb 2021 11:11:54 +0530 Subject: [PATCH 2/2] build pipeline changes and more tests --- .github/workflows/gobuild.yml | 3 + Makefile | 2 +- scripts/generate-coverage.sh | 2 +- test/e2e/init/config/invalid_path.toml | 2 + test/e2e/init/config/valid_config.toml | 3 + test/e2e/init/golden/init_help.txt | 2 +- test/e2e/init/init_test.go | 88 +++++++++++++++++--------- test/e2e/init/init_utils.go | 41 ++++++++++++ test/helper/helper.go | 18 +++--- tools/tools.go | 13 ++++ 10 files changed, 132 insertions(+), 42 deletions(-) create mode 100644 test/e2e/init/config/invalid_path.toml create mode 100644 test/e2e/init/config/valid_config.toml create mode 100644 test/e2e/init/init_utils.go create mode 100644 tools/tools.go diff --git a/.github/workflows/gobuild.yml b/.github/workflows/gobuild.yml index d950db96f..86e8197fe 100644 --- a/.github/workflows/gobuild.yml +++ b/.github/workflows/gobuild.yml @@ -30,6 +30,9 @@ jobs: - name: Run unit tests run: make unit-tests + - name: Run e2e tests + run: make e2e-tests + - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/Makefile b/Makefile index 3b82eaa0c..a9b621d28 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ cicd: validate build test docker-build # run all unit and integration tests -test: unit-tests +test: unit-tests e2e-tests # run all validation tests diff --git a/scripts/generate-coverage.sh b/scripts/generate-coverage.sh index 90978c878..ba3d1e1ee 100755 --- a/scripts/generate-coverage.sh +++ b/scripts/generate-coverage.sh @@ -4,5 +4,5 @@ set -o errexit set -o nounset set -o pipefail -go test -v -coverpkg=./... -coverprofile=coverage.out ./... +go test -v -coverpkg=./pkg/... -coverprofile=coverage.out ./pkg/... go tool cover -func coverage.out diff --git a/test/e2e/init/config/invalid_path.toml b/test/e2e/init/config/invalid_path.toml new file mode 100644 index 000000000..bcdcceaf4 --- /dev/null +++ b/test/e2e/init/config/invalid_path.toml @@ -0,0 +1,2 @@ +[policy] +path = "invalid/path" \ No newline at end of file diff --git a/test/e2e/init/config/valid_config.toml b/test/e2e/init/config/valid_config.toml new file mode 100644 index 000000000..f3f62657f --- /dev/null +++ b/test/e2e/init/config/valid_config.toml @@ -0,0 +1,3 @@ +[policy] +repo_url = "https://github.com/accurics/KaiMonkey.git" +branch = "master" \ No newline at end of file diff --git a/test/e2e/init/golden/init_help.txt b/test/e2e/init/golden/init_help.txt index 2195be35f..0a2d2a485 100644 --- a/test/e2e/init/golden/init_help.txt +++ b/test/e2e/init/golden/init_help.txt @@ -12,4 +12,4 @@ Global Flags: -c, --config-path string config file path -l, --log-level string log level (debug, info, warn, error, panic, fatal) (default "info") -x, --log-type string log output type (console, json) (default "console") - -o, --output string output type (human, json, yaml, xml) (default "human") + -o, --output string output type (human, json, yaml, xml, junit-xml) (default "human") diff --git a/test/e2e/init/init_test.go b/test/e2e/init/init_test.go index added5313..818d34655 100644 --- a/test/e2e/init/init_test.go +++ b/test/e2e/init/init_test.go @@ -4,7 +4,9 @@ import ( "io" "os" "path/filepath" + "time" + initUtil "github.com/accurics/terrascan/test/e2e/init" "github.com/accurics/terrascan/test/helper" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -13,15 +15,13 @@ import ( "gopkg.in/src-d/go-git.v4" ) -const ( - initCommandTimeout = 60 -) - var ( + initCommand string = "init" defaultPolicyRepoPath string = os.Getenv("HOME") + "/.terrascan" terrascanGitURL string = "https://github.com/accurics/terrascan.git" terrascanDefaultBranch string = "master" terrascanConfigEnvName string = "TERRASCAN_CONFIG" + kaiMoneyGitURL string = "https://github.com/accurics/KaiMonkey.git" ) var _ = Describe("Init", func() { @@ -46,10 +46,9 @@ var _ = Describe("Init", func() { }) Describe("terrascan init is run", func() { - When("terrascan init is run without any flags", func() { + When("without any flags", func() { It("should download policies and exit with status code 0", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") - Eventually(session, initCommandTimeout).Should(gexec.Exit(0)) + session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) Expect(outWriter).Should(gbytes.Say("")) }) @@ -61,19 +60,10 @@ var _ = Describe("Init", func() { var repo *git.Repository var err error It("should be a valid git repo", func() { - repo, err = git.PlainOpen(defaultPolicyRepoPath) - Expect(err).NotTo(HaveOccurred()) - Expect(repo).NotTo(BeNil()) + repo = initUtil.OpenGitRepo(defaultPolicyRepoPath) }) It("should be terrascan git repo", func() { - remote, err := repo.Remote("origin") - Expect(err).NotTo(HaveOccurred()) - Expect(remote).NotTo(BeNil()) - remoteConfig := remote.Config() - Expect(remoteConfig).NotTo(BeNil()) - err = remoteConfig.Validate() - Expect(err).NotTo(HaveOccurred()) - Expect(remoteConfig.URLs[0]).To(BeEquivalentTo(terrascanGitURL)) + initUtil.ValidateGitRepo(repo, terrascanGitURL) }) It("master branch should be present", func() { _, err = repo.Branch(terrascanDefaultBranch) @@ -84,14 +74,14 @@ var _ = Describe("Init", func() { When("terrascan init is run with -h flag", func() { It("should print help", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init", "-h") + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, initCommand, "-h") goldenFileAbsPath, err := filepath.Abs("golden/init_help.txt") Expect(err).NotTo(HaveOccurred()) helper.CompareActualWithGolden(session, goldenFileAbsPath, true) }) It("should exit with status code 0", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init", "-h") + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, initCommand, "-h") Eventually(session).Should(gexec.Exit(0)) }) }) @@ -128,9 +118,8 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") - Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) - helper.ContainsErrorSubString(session, `failed to download policies. error: 'Get "https://repository/url/info/refs?service=git-upload-pack": dial tcp: lookup repository on 8.8.8.8:53: no such host'`) + session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) + helper.ContainsErrorSubString(session, `failed to download policies. error: 'Get "https://repository/url/info/refs?service=git-upload-pack": dial tcp:`) }) }) When("the config file has invalid branch name", func() { @@ -141,9 +130,8 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") - Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) - helper.ContainsErrorSubString(session, `failed to checkout branch 'invalid-branch'. error: 'reference not found'`) + session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) + helper.ContainsErrorSubString(session, `failed to initialize terrascan. error : failed to checkout git branch 'invalid-branch'. error: 'reference not found'`) }) }) When("the config file has invalid rego subdir", func() { @@ -154,8 +142,7 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") - Eventually(session, initCommandTimeout).Should(gexec.Exit(1)) + session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) helper.ContainsErrorSubString(session, "invalid/path: no such file or directory") }) }) @@ -166,8 +153,49 @@ var _ = Describe("Init", func() { JustAfterEach(func() { os.Setenv(terrascanConfigEnvName, "") }) - It("should error out and exit with status code 1", func() { - Skip("Skipping invalid path test until discussion with team") + It("should should download policies and exit with status code 0", func() { + initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + }) + }) + Context("the config file has valid data", func() { + When("config file has different git repo and branch", func() { + JustBeforeEach(func() { + os.Setenv(terrascanConfigEnvName, "config/valid_config.toml") + }) + JustAfterEach(func() { + os.Setenv(terrascanConfigEnvName, "") + }) + It("init should download the repo provided in the config file", func() { + initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + }) + Context("Kai Monkey git repo is downloaded", func() { + It("should validate Kai Monkey repo in the policy path", func() { + repo := initUtil.OpenGitRepo(defaultPolicyRepoPath) + initUtil.ValidateGitRepo(repo, kaiMoneyGitURL) + }) + }) + }) + }) + }) + + Describe("terrascan init is run multiple times", func() { + Context("init clones the git repo to a temp dir, deletes policy path and renames tempdir to policy path", func() { + Context("running init the first time", func() { + var modifiedTime time.Time + It("should download policies at the default policy path", func() { + initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + fi, err := os.Stat(defaultPolicyRepoPath) + Expect(err).ToNot(HaveOccurred()) + modifiedTime = fi.ModTime() + }) + Context("running init the second time", func() { + It("should download policies again at the default policy path", func() { + initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + fi, err := os.Stat(defaultPolicyRepoPath) + Expect(err).ToNot(HaveOccurred()) + Expect(fi.ModTime()).To(BeTemporally(">", modifiedTime)) + }) + }) }) }) }) diff --git a/test/e2e/init/init_utils.go b/test/e2e/init/init_utils.go new file mode 100644 index 000000000..b57003b36 --- /dev/null +++ b/test/e2e/init/init_utils.go @@ -0,0 +1,41 @@ +package init + +import ( + "io" + + "github.com/accurics/terrascan/test/helper" + "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + "gopkg.in/src-d/go-git.v4" +) + +const ( + initCommandTimeout = 60 +) + +// RunInitCommand will execute the init command and verify exit code +func RunInitCommand(terrascanBinaryPath string, outWriter, errWriter io.Writer, exitCode int) *gexec.Session { + session := helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "init") + gomega.Eventually(session, initCommandTimeout).Should(gexec.Exit(exitCode)) + return session +} + +// OpenGitRepo checks if a directory is a git repo +func OpenGitRepo(repoPath string) *git.Repository { + repo, err := git.PlainOpen(repoPath) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(repo).NotTo(gomega.BeNil()) + return repo +} + +// ValidateGitRepo validates a git repo and verifies the git url +func ValidateGitRepo(repo *git.Repository, gitURL string) { + remote, err := repo.Remote("origin") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(remote).NotTo(gomega.BeNil()) + remoteConfig := remote.Config() + gomega.Expect(remoteConfig).NotTo(gomega.BeNil()) + err = remoteConfig.Validate() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(remoteConfig.URLs[0]).To(gomega.BeEquivalentTo(gitURL)) +} diff --git a/test/helper/helper.go b/test/helper/helper.go index 9eae13800..88b77c0ae 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -6,33 +6,33 @@ import ( "os" "os/exec" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" ) // CompareActualWithGolden compares func CompareActualWithGolden(session *gexec.Session, goldenFileAbsPath string, isStdOut bool) { fileData, err := ioutil.ReadFile(goldenFileAbsPath) - Expect(err).NotTo(HaveOccurred()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) if isStdOut { - Expect(string(session.Wait().Out.Contents())).Should(BeIdenticalTo(string(fileData))) + gomega.Expect(string(session.Wait().Out.Contents())).Should(gomega.BeIdenticalTo(string(fileData))) } else { - Expect(string(session.Wait().Err.Contents())).Should(BeIdenticalTo(string(fileData))) + gomega.Expect(string(session.Wait().Err.Contents())).Should(gomega.BeIdenticalTo(string(fileData))) } } // ContainsErrorSubString will assert if error string is part of error output func ContainsErrorSubString(session *gexec.Session, errSubString string) { - Expect(string(session.Wait().Err.Contents())).Should(ContainSubstring(errSubString)) + gomega.Expect(string(session.Wait().Err.Contents())).Should(gomega.ContainSubstring(errSubString)) } // GetTerrascanBinaryPath returns the terrascan binary path func GetTerrascanBinaryPath() string { terrascanBinaryPath := os.Getenv("TERRASCAN_BIN_PATH") - Describe("terrascan binary path should be set for executing tests", func() { + ginkgo.Describe("terrascan binary path should be set for executing tests", func() { if terrascanBinaryPath == "" { - Fail("ensure that TERRASCAN_BIN_PATH is set") + ginkgo.Fail("ensure that TERRASCAN_BIN_PATH is set") } }) return terrascanBinaryPath @@ -42,6 +42,6 @@ func GetTerrascanBinaryPath() string { func RunCommand(path string, outWriter, errWriter io.Writer, args ...string) *gexec.Session { cmd := exec.Command(path, args...) session, err := gexec.Start(cmd, outWriter, errWriter) - Expect(err).NotTo(HaveOccurred()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) return session } diff --git a/tools/tools.go b/tools/tools.go new file mode 100644 index 000000000..8cf1325b6 --- /dev/null +++ b/tools/tools.go @@ -0,0 +1,13 @@ +// +build tools + +package tools + +import ( + // used only for testing + _ "github.com/onsi/ginkgo/ginkgo" + // used only for testing + _ "github.com/onsi/gomega" +) + +// This file imports packages that are used when running go generate, or used +// during the development process but not otherwise depended on by built code.