diff --git a/server/controllers/events/events_controller_e2e_test.go b/server/controllers/events/events_controller_e2e_test.go index 45d68de0ce..7965a50e27 100644 --- a/server/controllers/events/events_controller_e2e_test.go +++ b/server/controllers/events/events_controller_e2e_test.go @@ -42,7 +42,9 @@ import ( . "github.com/runatlantis/atlantis/testing" ) -const ConftestVersion = "0.36.0" // renovate: datasource=github-releases depName=open-policy-agent/conftest +// In the e2e test, we use `conftest` not `conftest$version`. +// Because if depends on the version, we need to upgrade test base image before e2e fix it. +const conftestCommand = "conftest" var applyLocker locking.ApplyLocker var userConfig server.UserConfig @@ -64,8 +66,8 @@ func (m *NoopTFDownloader) GetAny(dst, src string, opts ...getter.ClientOption) type LocalConftestCache struct { } -func (m *LocalConftestCache) Get(key *version.Version) (string, error) { - return exec.LookPath(fmt.Sprintf("conftest%s", ConftestVersion)) +func (m *LocalConftestCache) Get(_ *version.Version) (string, error) { + return exec.LookPath(conftestCommand) } func TestGitHubWorkflow(t *testing.T) { @@ -886,13 +888,6 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers. allowForkPRs := false dataDir, binDir, cacheDir := mkSubDirs(t) - //env vars - - if userConfig.EnablePolicyChecksFlag { - // need this to be set or we'll fail the policy check step - os.Setenv(policy.DefaultConftestVersionEnvKey, "0.25.0") - } - // Mocks. e2eVCSClient := vcsmocks.NewMockClient() e2eStatusUpdater := &events.DefaultCommitStatusUpdater{Client: e2eVCSClient} @@ -1016,8 +1011,6 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers. Ok(t, err) - conftestVersion, _ := version.NewVersion(ConftestVersion) - conftextExec := policy.NewConfTestExecutorWorkflow(logger, binDir, &NoopTFDownloader{}) // swapping out version cache to something that always returns local contest @@ -1025,7 +1018,7 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers. conftextExec.VersionCache = &LocalConftestCache{} policyCheckRunner, err := runtime.NewPolicyCheckStepRunner( - conftestVersion, + defaultTFVersion, conftextExec, ) @@ -1392,30 +1385,16 @@ func mkSubDirs(t *testing.T) (string, string, string) { return tmp, binDir, cachedir } -// Will fail test if conftest isn't in path or is version less than specific version +// Will fail test if conftest isn't in path func ensureRunningConftest(t *testing.T) { - localPath, err := exec.LookPath("conftest") + // use `conftest` command instead `contest$version`, so tests may fail on the environment cause the output logs may become change by version. + t.Logf("conftest check may fail depends on conftest version. please use latest stable conftest.") + _, err := exec.LookPath(conftestCommand) if err != nil { - t.Logf("conftest must be installed to run this test") - t.FailNow() - } - versionOutBytes, err := exec.Command(localPath, "--version").Output() // #nosec - if err != nil { - t.Logf("error running conftest version: %s", err) - t.FailNow() - } - versionOutput := string(versionOutBytes) - match := versionConftestRegex.FindStringSubmatch(versionOutput) - if len(match) <= 1 { - t.Logf("could not parse conftest version from %s", versionOutput) - t.FailNow() - } - localVersion, err := version.NewVersion(match[1]) - Ok(t, err) - minVersion, err := version.NewVersion(ConftestVersion) - Ok(t, err) - if localVersion.LessThan(minVersion) { - t.Logf("must have contest version >= %s, you have %s", minVersion, localVersion) + t.Logf(`%s must be installed to run this test +- on local, please install contest command or run 'make docker/test-all' +- on CI, please check testing-env docker image contains conftest command. see testing/Dockerfile +`, conftestCommand) t.FailNow() } } diff --git a/server/core/runtime/policy/conftest_client.go b/server/core/runtime/policy/conftest_client.go index 08a5ee297a..8a7ffad561 100644 --- a/server/core/runtime/policy/conftest_client.go +++ b/server/core/runtime/policy/conftest_client.go @@ -211,9 +211,13 @@ func (c *ConfTestExecutorWorkflow) sanitizeOutput(inputFile string, output strin } func (c *ConfTestExecutorWorkflow) EnsureExecutorVersion(log logging.SimpleLogging, v *version.Version) (string, error) { - // we have no information to proceed so fail hard + // we have no information to proceed, so fallback to `conftest` command or fail hard if c.DefaultConftestVersion == nil && v == nil { - return "", errors.New("no conftest version configured/specified") + localPath, err := c.Exec.LookPath(conftestBinaryName) + if err == nil { + return localPath, nil + } + return "", errors.New("no conftest version configured/specified or not found conftest command") } var versionToRetrieve *version.Version diff --git a/server/core/runtime/policy/conftest_client_test.go b/server/core/runtime/policy/conftest_client_test.go index d90af16c49..4483786823 100644 --- a/server/core/runtime/policy/conftest_client_test.go +++ b/server/core/runtime/policy/conftest_client_test.go @@ -64,18 +64,32 @@ func TestEnsureExecutorVersion(t *testing.T) { RegisterMockTestingT(t) mockCache := mocks.NewMockExecutionVersionCache() + mockExec := models_mocks.NewMockExec() log := logging.NewNoopLogger(t) - t.Run("no specified version or default version", func(t *testing.T) { + t.Run("no specified version or default version without conftest command", func(t *testing.T) { subject := &ConfTestExecutorWorkflow{ VersionCache: mockCache, + Exec: mockExec, } + When(mockExec.LookPath(AnyString())).ThenReturn("", errors.New("not found")) _, err := subject.EnsureExecutorVersion(log, nil) Assert(t, err != nil, "expected error finding version") }) + t.Run("no specified version or default version with conftest command", func(t *testing.T) { + subject := &ConfTestExecutorWorkflow{ + VersionCache: mockCache, + Exec: mockExec, + } + When(mockExec.LookPath(AnyString())).ThenReturn(expectedPath, nil) + path, err := subject.EnsureExecutorVersion(log, nil) + Ok(t, err) + Assert(t, path == expectedPath, "path is expected") + }) + t.Run("use default version", func(t *testing.T) { subject := &ConfTestExecutorWorkflow{ VersionCache: mockCache, diff --git a/testing/Dockerfile b/testing/Dockerfile index 9b968f43ff..7bcb47bca8 100644 --- a/testing/Dockerfile +++ b/testing/Dockerfile @@ -24,8 +24,8 @@ RUN case $(uname -m) in x86_64|amd64) ARCH="x86_64" ;; aarch64|arm64|armv7l) ARC sed -n "/conftest_${CONFTEST_VERSION}_Linux_${ARCH}.tar.gz/p" checksums.txt | sha256sum -c && \ mkdir -p /usr/local/bin/cft/versions/${CONFTEST_VERSION} && \ tar -C /usr/local/bin/cft/versions/${CONFTEST_VERSION} -xzf conftest_${CONFTEST_VERSION}_Linux_${ARCH}.tar.gz && \ - # After migrating e2e test which use contest command instead of contest$version, you can remove contest$version command. - ln -s /usr/local/bin/cft/versions/${CONFTEST_VERSION}/conftest /usr/local/bin/conftest${CONFTEST_VERSION} && \ + # Generally Atlantis requires `conftest$version` command. But we use `conftest` command in test. + # `conftest$version` command blocks upgrading conftest operation cause e2e test use this image. ln -s /usr/local/bin/cft/versions/${CONFTEST_VERSION}/conftest /usr/local/bin/conftest && \ rm conftest_${CONFTEST_VERSION}_Linux_${ARCH}.tar.gz && \ rm checksums.txt