diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go
index 7bdf71e6b8..4ea8807cca 100644
--- a/test/e2e/conformance_test.go
+++ b/test/e2e/conformance_test.go
@@ -57,6 +57,7 @@ var _ = Describe("conformance tests", func() {
 	Measure(specName, func(b Benchmarker) {
 
 		name := fmt.Sprintf("cluster-%s", util.RandomString(6))
+		setEnvVar("USE_CI_ARTIFACTS", "true", false)
 		kubernetesVersion := e2eConfig.GetVariable(KubernetesVersion)
 		flavor := clusterctl.DefaultFlavor
 		if useCIArtifacts {
@@ -105,6 +106,7 @@ var _ = Describe("conformance tests", func() {
 	}, 1)
 
 	AfterEach(func() {
+		setEnvVar("USE_CI_ARTIFACTS", "false", false)
 		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
 		dumpSpecResourcesAndCleanup(ctx, "", bootstrapClusterProxy, artifactFolder, namespace, e2eConfig.GetIntervals, skipCleanup)
 	})
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index c3ad828aea..300cff1c46 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -174,13 +174,24 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	Expect(err).NotTo(HaveOccurred())
 	platformKustomization, err := ioutil.ReadFile("data/ci-artifacts-platform-kustomization.yaml")
 	Expect(err).NotTo(HaveOccurred())
-	_, err = kubernetesversions.GenerateCIArtifactsInjectedTemplateForDebian(
+	ciTemplate, err := kubernetesversions.GenerateCIArtifactsInjectedTemplateForDebian(
 		kubernetesversions.GenerateCIArtifactsInjectedTemplateForDebianInput{
 			ArtifactsDirectory:    artifactFolder,
 			SourceTemplate:        sourceTemplate,
 			PlatformKustomization: platformKustomization,
 		},
 	)
+	clusterctlCITemplate := clusterctl.Files{
+		SourcePath: ciTemplate,
+		TargetName: "cluster-template-conformance-ci-artifacts.yaml",
+	}
+	providers := e2eConfig.Providers
+	for i, prov := range providers {
+		if prov.Name != "aws" {
+			continue
+		}
+		e2eConfig.Providers[i].Files = append(e2eConfig.Providers[i].Files, clusterctlCITemplate)
+	}
 	Expect(err).NotTo(HaveOccurred())
 	awsSession = newAWSSession()
 	createCloudFormationStack(awsSession, getBootstrapTemplate())