Support custom e2e kustomization resources (#1294)

* support custom e2e kustomization

* support list of resources

* fix image name/tag

* bump dependencies

* update vars

* add ecr creds helper

* enable e2e tests on merge commits to main

* bump test-infra-defs build image

* remove toolchain from go.mod

* disable e2e on mergequeue

* update cluster tag

* fix cluster name

* increase test waitFor duration

* update e2e go version

---------

Co-authored-by: levan-m <[email protected]>

.gitlab-ci.yml

@@ -14,7 +14,7 @@ variables:
   RH_PARTNER_REGISTRY_USER: "redhat-isv-containers+5e7c8ebc1c86a3163d1a69be-robot"
   RH_PARTNER_REGISTRY_KEY_SSM_KEY: redhat_registry_key
   RH_PARTNER_API_KEY_SSM_KEY: redhat_api_key
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: a1d921006e35
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 3c7d2dc2d3dd
   PUSH_IMAGES_TO_STAGING:
     description: "Set PUSH_IMAGE_TO_STAGING to 'true' if you want to push the operator to internal staging registry."
 
@@ -56,14 +56,16 @@ stages:
     KUBERNETES_MEMORY_LIMIT: 16Gi
 
 .on_run_e2e:
-  - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
+  - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'
     when: always
   - if: $CI_COMMIT_BRANCH
     changes:
       paths:
         - "*.md"
       compare_to: "refs/heads/main"
     when: never
+  - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
+    when: never
   # Temporarily disable on Conductor-triggered jobs
   - if: '$DDR != "true"'
     when: manual

Makefile

@@ -210,14 +210,10 @@ integration-tests-v2: $(ENVTEST) ## Run tests with reconciler V2
 
 .PHONY: e2e-tests
 e2e-tests: manifests $(KUSTOMIZE) ## Run E2E tests and destroy environment stacks after tests complete. To run locally, complete pre-reqs (see docs/how-to-contribute.md) and prepend command with `aws-vault exec sso-agent-sandbox-account-admin --`. E.g. `aws-vault exec sso-agent-sandbox-account-admin -- make e2e-tests`.
-	cd config/e2e && $(ROOT)/$(KUSTOMIZE) edit set image controller=$(IMG)
-	$(KUSTOMIZE) build config/e2e
 	KUBEBUILDER_ASSETS="$(ROOT)/bin/$(PLATFORM)/" go test -C test/e2e --tags=e2e github.com/DataDog/datadog-operator/e2e -v -timeout 1h -coverprofile cover_e2e.out
 
 .PHONY: e2e-tests-keep-stacks
 e2e-tests-keep-stacks: manifests $(KUSTOMIZE) ## Run E2E tests and keep environment stacks running. To run locally, complete pre-reqs (see docs/how-to-contribute.md) and prepend command with `aws-vault exec sso-agent-sandbox-account-admin --`. E.g. `aws-vault exec sso-agent-sandbox-account-admin -- make e2e-tests-keep-stacks`.
-	cd config/e2e && $(ROOT)/$(KUSTOMIZE) edit set image controller=$(IMG)
-	$(KUSTOMIZE) build config/e2e
 	KUBEBUILDER_ASSETS="$(ROOT)/bin/$(PLATFORM)/" go test -C test/e2e --tags=e2e github.com/DataDog/datadog-operator/e2e -v -timeout 1h -coverprofile cover_e2e_keep_stacks.out -args -keep-stacks=true
 
 .PHONY: bundle

test/e2e/common_test.go → test/e2e/common.go

@@ -6,18 +6,26 @@
 package e2e
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+
+	"github.com/DataDog/datadog-operator/pkg/plugin/common"
 )
 
 const (
 	manifestsPath       = "./manifests"
 	mgrKustomizeDirPath = "../../config/e2e"
+	defaultMgrImageName = "gcr.io/datadoghq/operator"
+	defaultMgrImgTag    = "latest"
+	defaultMgrFileName  = "e2e-manager.yaml"
 )
 
 var (
@@ -97,3 +105,75 @@ func deleteDda(t *testing.T, kubectlOptions *k8s.KubectlOptions, ddaPath string)
 		k8s.KubectlDelete(t, kubectlOptions, ddaPath)
 	}
 }
+
+func loadKustomization(path string) (*types.Kustomization, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var kustomization types.Kustomization
+	if err := yaml.Unmarshal(data, &kustomization); err != nil {
+		return nil, err
+	}
+
+	return &kustomization, nil
+}
+
+func saveKustomization(path string, kustomization *types.Kustomization) error {
+	data, err := yaml.Marshal(kustomization)
+	if err != nil {
+		return err
+	}
+
+	if err := os.WriteFile(path, data, 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// updateKustomization Updates kustomization.yaml file in given kustomize directory with extra resources and image name and tag if `IMG` environment variable is set.
+func updateKustomization(kustomizeDirPath string, kustomizeResourcePaths []string) error {
+	var imgName, imgTag string
+
+	kustomizationFilePath := fmt.Sprintf("%s/kustomization.yaml", kustomizeDirPath)
+	k, err := loadKustomization(kustomizationFilePath)
+	if err != nil {
+		return err
+	}
+
+	// Update resources with target e2e-manager resource yaml
+	for _, res := range kustomizeResourcePaths {
+		exists := false
+		for _, r := range k.Resources {
+			if r == res {
+				exists = true
+				break
+			}
+		}
+		if !exists {
+			k.Resources = append(k.Resources, res)
+		}
+	}
+
+	// Update image
+	if os.Getenv("IMG") != "" {
+		imgName, imgTag = common.SplitImageString(os.Getenv("IMG"))
+	} else {
+		imgName = defaultMgrImageName
+		imgTag = defaultMgrImgTag
+	}
+	for i, img := range k.Images {
+		if img.Name == "controller" {
+			k.Images[i].NewName = imgName
+			k.Images[i].NewTag = imgTag
+		}
+	}
+
+	if err := saveKustomization(kustomizationFilePath, k); err != nil {
+		return err
+	}
+
+	return nil
+}