[8.4](backport #3285) fix: remove Helm Chart tests

.ci/.e2e-tests-beats.yaml

@@ -1,14 +1,5 @@
 ---
 SUITES:
-  - suite: "helm"
-    provider: "docker"
-    scenarios:
-      - name: "Filebeat"
-        tags: "filebeat"
-        platforms: ["debian_10_amd64"]
-      - name: "Metricbeat"
-        tags: "metricbeat"
-        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"

.ci/.e2e-tests-daily.yaml

@@ -1,17 +1,5 @@
 ---
 SUITES:
-  - suite: "helm"
-    provider: "docker"
-    scenarios:
-      - name: "APM Server"
-        tags: "apm-server"
-        platforms: ["debian_10_amd64"]
-      - name: "Filebeat"
-        tags: "filebeat"
-        platforms: ["debian_10_amd64"]
-      - name: "Metricbeat"
-        tags: "metricbeat"
-        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"

.ci/.e2e-tests.yaml

@@ -1,17 +1,5 @@
 ---
 SUITES:
-  - suite: "helm"
-    provider: "docker"
-    scenarios:
-      - name: "APM Server"
-        tags: "apm-server"
-        platforms: ["debian_10_amd64"]
-      - name: "Filebeat"
-        tags: "filebeat"
-        platforms: ["debian_10_amd64"]
-      - name: "Metricbeat"
-        tags: "metricbeat"
-        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"

.ci/Jenkinsfile

@@ -58,7 +58,7 @@ pipeline {
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
     string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.4.4-dcd4a95f-SNAPSHOT', description: 'SemVer version of the Elastic Agent to be used for the tests. You can use here the tag of your PR to test your changes')
     string(name: 'BEAT_VERSION', defaultValue: '8.4.4-dcd4a95f-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes')
-    choice(name: 'LOG_LEVEL', choices: ['TRACE', 'DEBUG', 'INFO'], description: 'Log level to be used')
+    choice(name: 'LOG_LEVEL', choices: ['INFO', 'DEBUG', 'TRACE'], description: 'Log level to be used')
     choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
     string(name: 'KIBANA_VERSION', defaultValue: '', description: 'Docker tag of the kibana to be used for the tests. It will refer to an image related to a Kibana PR, under the Observability-CI namespace')
     string(name: 'STACK_VERSION', defaultValue: '8.4.4-dcd4a95f-SNAPSHOT', description: 'SemVer version of the stack to be used for the tests.')
@@ -217,17 +217,9 @@ pipeline {
                     "STACK_INSTANCE_ID=${env.BUILD_URL}_stack",
                     "TAGS=non-existing-tag"
                   ]) {
-                    ciBuild() {
-                      sh(label: 'Create Stack node', script: "make -C .ci provision-stack")
-                    }
                     ciBuild() {
                       retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Setup Stack node', script: "make -C .ci setup-stack")
-                      }
-                    }
-                    ciBuild() {
-                      retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Start Elastic Stack', script: "make -C .ci start-elastic-stack")
+                        sh(label: 'Setup Stack node', script: "make -C .ci create-stack")
                       }
                     }
                   }
@@ -281,15 +273,27 @@ pipeline {
             cleanup {
               // Once all tests are complete we need to teardown the single instance with the deployed stack
               script {
+                dir("${env.REAL_BASE_DIR}") {
+                  ciBuild() {
+                    def stackIP = getNodeIp('stack')
+                    sh(label: 'Grab logs', script:"make -C .ci fetch-test-reports NODE_IP_ADDRESS=${stackIP} NODE_LABEL=debian_10_amd64")
+                    archiveArtifacts(allowEmptyArchive: true, artifacts: "outputs/**/TEST-*,outputs/**/*.zip,outputs/**/*.tgz")
+                    junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/**/TEST-*.xml")
+                  }
+                }
                 def stackMachine = getMachineInfo('stack')
                 if (!params.DESTROY_CLOUD_RESOURCES) {
                   def stackRunnerIP = getNodeIp('stack')
                   log(level: 'DEBUG', text: "Stack instance won't be destroyed after the build. Please SSH into the stack machine on ${stackRunnerIP}")
                 } else {
                   dir("${env.REAL_BASE_DIR}") {
-                    ciBuild() {
-                      retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Destroy stack node', script: "make -C .ci destroy-stack")
+                    withEnv([
+                      "STACK_INSTANCE_ID=${env.BUILD_URL}_stack",
+                    ]) {
+                      ciBuild() {
+                        retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                          sh(label: 'Destroy stack node', script: "make -C .ci destroy-stack")
+                        }
                       }
                     }
                   }
@@ -338,7 +342,7 @@ def checkSkipTests() {
     }
 
     // patterns for all places that should trigger a full build
-    def regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/helm/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
+    def regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
     setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: regexps, shouldMatchAll: false))
   }
 }
@@ -362,9 +366,6 @@ def ciBuild(Closure body){
           [var: "AWS_SECRET_ACCESS_KEY", password: awsAuthObj.secret_key]
         ]) {
           withOtelEnv() {
-            retryWithSleep(retries: 3, seconds: 5, backoff: true){
-              sh("make -C .ci setup-env") // make sure the environment is created
-            }
             body()
           }
         }
@@ -603,21 +604,8 @@ def generateFunctionalTestStep(Map args = [:]){
                 }
               }
             }
-            withEnv([
-              "ARCHITECTURE=${goArch}",
-              "CUCUMBER_REPORTS_PATH=${env.REAL_BASE_DIR}/outputs/${testRunnerIP}",
-              "PLATFORM=${platform}",
-              "SUITE=${suite}",
-              "TAGS=${tags}",
-            ]){
-              retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-                sh(script: ".ci/scripts/generate-cucumber-reports.sh", label: "generate-cucumber-reports.sh")
-              }
-            }
-            junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/${testRunnerIP}/TEST-*.xml")
-            archiveArtifacts allowEmptyArchive: true,
-              artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html"
+            archiveArtifacts(allowEmptyArchive: true, artifacts: "outputs/**/TEST-*,outputs/**/*.zip,outputs/**/*.tgz")
+            junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/**/TEST-*.xml")
           }
         }
       }

.ci/Makefile

@@ -24,7 +24,7 @@ STACK_INSTANCE_ID ?= stack_$(STACK_LABEL)_$(RUN_ID)
 # Debian 10 AMD (see .e2e-platforms.yaml)
 NODE_IMAGE ?= ami-0d90bed76900e679a
 NODE_INSTANCE_TYPE ?= t3.xlarge
-NODE_LABEL ?= debian_amd64
+NODE_LABEL ?= debian_10_amd64
 NODE_SHELL_TYPE ?= sh
 NODE_USER ?= admin
 NODE_IP_ADDRESS ?= $(shell cat $(PROJECT_DIR)/.ci/.node-host-ip)
@@ -41,6 +41,12 @@ SUITE ?= fleet
 # Tags to run. Please check out the feature files
 TAGS ?= fleet_mode
 
+SHELL = /bin/bash
+MAKEFLAGS += --silent --no-print-directory
+.SHELLFLAGS = -ec
+
+export ANSIBLE_CONFIG := $(CURDIR)/ansible/ansible.cfg
+
 # Check that given variables are set and all have non-empty values,
 # die with an error otherwise.
 #
@@ -61,6 +67,21 @@ __check_defined = \
 .runID:
 	echo "$(shell uuidgen|cut -d'-' -f1)" > $(RUN_ID_FILE)
 
+.PHONY: show-env
+show-env:
+	@source $(VENV_DIR)/bin/activate; \
+	echo "PROJECT_DIR: $(PROJECT_DIR)"; \
+	echo "VENV_DIR: $(VENV_DIR)"; \
+	echo "VENV_BIN_DIR: $(VENV_BIN_DIR)"; \
+	echo "ANSIBLE: $$(ansible --version)"; \
+	echo "Python: $$(python --version)"; \
+	echo "RUN_ID: $(RUN_ID)"; \
+	echo "STACK_VERSION: $(STACK_VERSION)"; \
+	echo "SUITE: $(SUITE)"; \
+	echo "TAGS: $(TAGS)"; \
+	echo "PROVIDER: $(PROVIDER)";
+	echo "NODE_LABEL: $(NODE_LABEL)";
+
 .PHONY: setup-env
 setup-env:
 	$(MAKE) .runID
@@ -74,17 +95,18 @@ destroy-env:
 	rm -fr $(VENV_DIR)
 
 .PHONY: list-platforms
-list-platforms:
+list-platforms: 
 	@docker run --rm -i -w "/workdir" -v $(PROJECT_DIR)/.ci:/workdir mikefarah/yq:4 ".PLATFORMS | keys" .e2e-platforms.yaml
 
 # Create the env file for the target platform
 .PHONY: set-env-%
 set-env-%:
-	@$(PROJECT_DIR)/.ci/scripts/yq.sh "$*"
+	@$(PROJECT_DIR)/.ci/scripts/gen-platform-env-file.py "$*"
 
 .PHONY: provision-stack
-provision-stack: .runID
+provision-stack: setup-env show-env
 	@:$(call check_defined, RUN_ID, You need to an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE) nodeUser=$(STACK_USER)" \
@@ -94,9 +116,10 @@ provision-stack: .runID
 
 .PHONY: setup-stack
 setup-stack: export TAGS = non-existing-tag
-setup-stack: .runID
+setup-stack: setup-env show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
+	source $(VENV_DIR)/bin/activate; \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE) nodeUser=$(STACK_USER)" \
@@ -109,15 +132,15 @@ setup-stack: .runID
 create-stack: provision-stack setup-stack start-elastic-stack
 
 .PHONY: destroy-stack
-destroy-stack:
+destroy-stack: setup-env show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
-	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
+	source $(VENV_DIR)/bin/activate; \
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/teardown.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars="$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeUser=$(STACK_IMAGE)" \
 		--extra-vars="runId=$(RUN_ID) instanceID=$(STACK_INSTANCE_ID) nodeShellType=$(STACK_SHELL_TYPE) workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
-		--ssh-common-args='$(SSH_OPTS)' \
-		-t destroy
-	rm -fr $(PROJECT_DIR)/.ci/.stack-host-ip
+		--ssh-common-args='$(SSH_OPTS)'
+	# rm -fr $(PROJECT_DIR)/.ci/.stack-host-ip
 
 .PHONY: ssh-stack
 ssh-stack:
@@ -132,9 +155,10 @@ show-stack:
 	@echo "Stack Shell         : $(STACK_SHELL_TYPE)"
 
 .PHONY: provision-node
-provision-node: .runID set-env-$(NODE_LABEL)
+provision-node: setup-env set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
@@ -143,11 +167,11 @@ provision-node: .runID set-env-$(NODE_LABEL)
 		-t provision-node
 
 .PHONY: setup-node
-setup-node: .runID set-env-$(NODE_LABEL)
+setup-node: setup-env set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
 	@:$(call check_defined, NODE_IP_ADDRESS, IP address of the test node not defined)
-	source $(VENV)/bin/activate; \
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
@@ -160,8 +184,9 @@ setup-node: .runID set-env-$(NODE_LABEL)
 create-node: provision-node setup-node
 
 .PHONY: destroy-node
-destroy-node: set-env-$(NODE_LABEL)
+destroy-node: setup-env set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/teardown.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars="$(LABELS_STRING) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeUser=$${NODE_USER}" \
@@ -170,9 +195,10 @@ destroy-node: set-env-$(NODE_LABEL)
 	rm -fr $(PROJECT_DIR)/.ci/.node-host-ip
 
 .PHONY: fetch-test-reports
-fetch-test-reports: .runID set-env-$(NODE_LABEL)
+fetch-test-reports: setup-env set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, NODE_IP_ADDRESS, IP address of the test node not defined)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/fetch-test-reports.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
@@ -199,8 +225,9 @@ destroy-elastic-stack:
 	ssh $(SSH_OPTS_EXTENDED) -i $(SSH_KEY) $(STACK_USER)@$(STACK_IP_ADDRESS) 'sudo docker-compose -f /root/.op/compose/profiles/fleet/docker-compose.yml down --remove-orphans'
 
 .PHONY: start-elastic-stack
-start-elastic-stack:
+start-elastic-stack: setup-env show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	PROVIDER="remote" SUITE="$(SUITE)" TAGS="non-existent-tag" \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/elastic-stack.yml \
 		--private-key="$(SSH_KEY)" \
@@ -217,8 +244,9 @@ recreate-fleet-server:
 	$(MAKE) start-elastic-stack
 
 .PHONY: run-tests
-run-tests: set-env-$(NODE_LABEL)
+run-tests: setup-env set-env-$(NODE_LABEL) show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && PROVIDER="$(PROVIDER)" SUITE="$(SUITE)" TAGS="$(TAGS)" REPORT_PREFIX="$(SUITE)_$${NODE_LABEL}_$(TAGS)" \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml \
 		--private-key="$(SSH_KEY)" \
@@ -230,8 +258,9 @@ run-tests: set-env-$(NODE_LABEL)
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && ssh $(SSH_OPTS_EXTENDED) -i $(SSH_KEY) $${NODE_USER}@$(NODE_IP_ADDRESS) "sudo bash /home/$${NODE_USER}/e2e-testing/.ci/scripts/functional-test.sh \"$(TAGS)\""
 
 .PHONY: run-tests-win
-run-tests-win: set-env-$(NODE_LABEL)
+run-tests-win: setup-env set-env-$(NODE_LABEL) show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && PROVIDER="$(PROVIDER)" SUITE="$(SUITE)" TAGS="$(TAGS)" REPORT_PREFIX="$(SUITE)_$${NODE_LABEL}_$(TAGS)" \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml \
 		--private-key="$(SSH_KEY)" \
@@ -241,3 +270,7 @@ run-tests-win: set-env-$(NODE_LABEL)
 		-t run-tests \
 		-i $(NODE_IP_ADDRESS),
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && ssh $(SSH_OPTS_EXTENDED) -i $(SSH_KEY) $${NODE_USER}@$(NODE_IP_ADDRESS) "powershell \"C:/Users/$${NODE_USER}/e2e-testing/.ci/scripts/functional-test.ps1\""
+
+.PHONY: clean
+clean:
+	rm -fr "$(PROJECT_DIR)/.ci/".env-* "$(PROJECT_DIR)/.ci/.node-host-ip" "$(PROJECT_DIR)/.ci/.runID" "$(PROJECT_DIR)/.ci/.stack-host-ip" "$(PROJECT_DIR)/outputs" "$(PROJECT_DIR)/None-sshhosts" "$(PROJECT_DIR)/stack-sshhosts"