fix: remove Helm Chart tests (#3285)

* bump stack version 8.7.0-04d5f080 * fix: remove Helm Chart tests * test: show env vars and versions * test: set shell flags * test: show env * fix: freeze requirements * docs: add ec2 issue * fix: issue with local pyenv versions installed * fix: improve health checks * fix: decrease debug level * fix: add some ansible config * feat: run filebeat to grab the Docker logs * fix: add run context * fix: stop stack * fix: grab docker logs * fix: stop stack * fix: grab junit results * fix: ignore errors * fix: typo * fix: grab stack logs from main agent * fix: env generation an configuration by default * docs: wrong steps * fix: update runners * fix: improve Stack provision * fix: remove attribute * fix: get public IP * fix: lint * fix: remove docker pull output * feat: add clean target * feat: gran Elastic Agent reports * fix: remove label setting * chore: Ansible confing * fix: set test logs right path * fix: retry stack deploy * feat: grab logs * fix: set proper label * feat: print logs * fix: increase retry time * test: change log to INFO * fix: sysntax error * fix: update update test versions * fix: grab logs * fix: remove folder before move agent * fix: change log level * fix: set command path * fix: fetch logs * fix: collect artifacts * feat: allow overwrite Elastic Stack config * fix: force binariy path * fix: test only releases * fix: report junit test results * fix: set runner Ip properly * test: collect logs in a tar file * fix: wrong flename * fix: remove fleet folder * fix: do not log env * fix: lint * fix: proper report folder name * feat: grab logs in fleet mode * fix: exclude more folders * fix: exclude elastic-agent binary * fix: update teardown playbook * fix: teardown the environment Co-authored-by: apmmachine <[email protected]> (cherry picked from commit d554138)
elastic · Jan 3, 2023 · e109cc4 · e109cc4
1 parent c4ef035
commit e109cc4
Show file tree

Hide file tree

Showing 56 changed files with 714 additions and 1,256 deletions.
diff --git a/.ci/.e2e-tests-beats.yaml b/.ci/.e2e-tests-beats.yaml
@@ -1,14 +1,5 @@
 ---
 SUITES:
-  - suite: "helm"
-    provider: "docker"
-    scenarios:
-      - name: "Filebeat"
-        tags: "filebeat"
-        platforms: ["debian_10_amd64"]
-      - name: "Metricbeat"
-        tags: "metricbeat"
-        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"

diff --git a/.ci/.e2e-tests-daily.yaml b/.ci/.e2e-tests-daily.yaml
@@ -1,17 +1,5 @@
 ---
 SUITES:
-  - suite: "helm"
-    provider: "docker"
-    scenarios:
-      - name: "APM Server"
-        tags: "apm-server"
-        platforms: ["debian_10_amd64"]
-      - name: "Filebeat"
-        tags: "filebeat"
-        platforms: ["debian_10_amd64"]
-      - name: "Metricbeat"
-        tags: "metricbeat"
-        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"

diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml
@@ -1,17 +1,5 @@
 ---
 SUITES:
-  - suite: "helm"
-    provider: "docker"
-    scenarios:
-      - name: "APM Server"
-        tags: "apm-server"
-        platforms: ["debian_10_amd64"]
-      - name: "Filebeat"
-        tags: "filebeat"
-        platforms: ["debian_10_amd64"]
-      - name: "Metricbeat"
-        tags: "metricbeat"
-        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"

diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -84,7 +84,6 @@ pipeline {
         KIBANA_VERSION = "${params.KIBANA_VERSION.trim()}"
         STACK_VERSION = "${params.STACK_VERSION.trim()}"
         FORCE_SKIP_GIT_CHECKS = "${params.forceSkipGitChecks}"
-        HELM_CHART_VERSION = "${params.HELM_CHART_VERSION.trim()}"
         HELM_VERSION = "${params.HELM_VERSION.trim()}"
         KIND_VERSION = "${params.KIND_VERSION.trim()}"
         KUBERNETES_VERSION = "${params.KUBERNETES_VERSION.trim()}"
@@ -157,17 +156,9 @@ pipeline {
                     "STACK_INSTANCE_ID=${env.BUILD_URL}_stack",
                     "TAGS=non-existing-tag"
                   ]) {
-                    ciBuild() {
-                      sh(label: 'Create Stack node', script: "make -C .ci provision-stack")
-                    }
                     ciBuild() {
                       retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Setup Stack node', script: "make -C .ci setup-stack")
-                      }
-                    }
-                    ciBuild() {
-                      retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Start Elastic Stack', script: "make -C .ci start-elastic-stack")
+                        sh(label: 'Setup Stack node', script: "make -C .ci create-stack")
                       }
                     }
                   }
@@ -221,15 +212,27 @@ pipeline {
             cleanup {
               // Once all tests are complete we need to teardown the single instance with the deployed stack
               script {
+                dir("${env.REAL_BASE_DIR}") {
+                  ciBuild() {
+                    def stackIP = getNodeIp('stack')
+                    sh(label: 'Grab logs', script:"make -C .ci fetch-test-reports NODE_IP_ADDRESS=${stackIP} NODE_LABEL=debian_10_amd64")
+                    archiveArtifacts(allowEmptyArchive: true, artifacts: "outputs/**/TEST-*,outputs/**/*.zip,outputs/**/*.tgz")
+                    junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/**/TEST-*.xml")
+                  }
+                }
                 def stackMachine = getMachineInfo('stack')
                 if (!params.DESTROY_CLOUD_RESOURCES) {
                   def stackRunnerIP = getNodeIp('stack')
                   log(level: 'DEBUG', text: "Stack instance won't be destroyed after the build. Please SSH into the stack machine on ${stackRunnerIP}")
                 } else {
                   dir("${env.REAL_BASE_DIR}") {
-                    ciBuild() {
-                      retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                        sh(label: 'Destroy stack node', script: "make -C .ci destroy-stack")
+                    withEnv([
+                      "STACK_INSTANCE_ID=${env.BUILD_URL}_stack",
+                    ]) {
+                      ciBuild() {
+                        retryWithSleep(retries: 3, seconds: 5, backoff: true){
+                          sh(label: 'Destroy stack node', script: "make -C .ci destroy-stack")
+                        }
                       }
                     }
                   }
@@ -278,7 +281,7 @@ def checkSkipTests() {
     }
 
     // patterns for all places that should trigger a full build
-    def regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/helm/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
+    def regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
     setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: regexps, shouldMatchAll: false))
   }
 }
@@ -302,9 +305,6 @@ def ciBuild(Closure body){
           [var: "AWS_SECRET_ACCESS_KEY", password: awsAuthObj.secret_key]
         ]) {
           withOtelEnv() {
-            retryWithSleep(retries: 3, seconds: 5, backoff: true){
-              sh("make -C .ci setup-env") // make sure the environment is created
-            }
             body()
           }
         }
@@ -543,21 +543,8 @@ def generateFunctionalTestStep(Map args = [:]){
                 }
               }
             }
-            withEnv([
-              "ARCHITECTURE=${goArch}",
-              "CUCUMBER_REPORTS_PATH=${env.REAL_BASE_DIR}/outputs/${testRunnerIP}",
-              "PLATFORM=${platform}",
-              "SUITE=${suite}",
-              "TAGS=${tags}",
-            ]){
-              retryWithSleep(retries: 3, seconds: 5, backoff: true){
-                dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-                sh(script: ".ci/scripts/generate-cucumber-reports.sh", label: "generate-cucumber-reports.sh")
-              }
-            }
-            junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/${testRunnerIP}/TEST-*.xml")
-            archiveArtifacts allowEmptyArchive: true,
-              artifacts: "outputs/${testRunnerIP}/TEST-*.xml, outputs/${testRunnerIP}/TEST-*.json, outputs/${testRunnerIP}/TEST-*.json.html"
+            archiveArtifacts(allowEmptyArchive: true, artifacts: "outputs/**/TEST-*,outputs/**/*.zip,outputs/**/*.tgz")
+            junit2otel(traceName: 'junit-e2e-tests', allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/**/TEST-*.xml")
           }
         }
       }

diff --git a/.ci/Makefile b/.ci/Makefile
@@ -24,7 +24,7 @@ STACK_INSTANCE_ID ?= stack_$(STACK_LABEL)_$(RUN_ID)
 # Debian 10 AMD (see .e2e-platforms.yaml)
 NODE_IMAGE ?= ami-0d90bed76900e679a
 NODE_INSTANCE_TYPE ?= t3.xlarge
-NODE_LABEL ?= debian_amd64
+NODE_LABEL ?= debian_10_amd64
 NODE_SHELL_TYPE ?= sh
 NODE_USER ?= admin
 NODE_IP_ADDRESS ?= $(shell cat $(PROJECT_DIR)/.ci/.node-host-ip)
@@ -41,6 +41,12 @@ SUITE ?= fleet
 # Tags to run. Please check out the feature files
 TAGS ?= fleet_mode
 
+SHELL = /bin/bash
+MAKEFLAGS += --silent --no-print-directory
+.SHELLFLAGS = -ec
+
+export ANSIBLE_CONFIG := $(CURDIR)/ansible/ansible.cfg
+
 # Check that given variables are set and all have non-empty values,
 # die with an error otherwise.
 #
@@ -61,6 +67,21 @@ __check_defined = \
 .runID:
 	echo "$(shell uuidgen|cut -d'-' -f1)" > $(RUN_ID_FILE)
 
+.PHONY: show-env
+show-env:
+	@source $(VENV_DIR)/bin/activate; \
+	echo "PROJECT_DIR: $(PROJECT_DIR)"; \
+	echo "VENV_DIR: $(VENV_DIR)"; \
+	echo "VENV_BIN_DIR: $(VENV_BIN_DIR)"; \
+	echo "ANSIBLE: $$(ansible --version)"; \
+	echo "Python: $$(python --version)"; \
+	echo "RUN_ID: $(RUN_ID)"; \
+	echo "STACK_VERSION: $(STACK_VERSION)"; \
+	echo "SUITE: $(SUITE)"; \
+	echo "TAGS: $(TAGS)"; \
+	echo "PROVIDER: $(PROVIDER)";
+	echo "NODE_LABEL: $(NODE_LABEL)";
+
 .PHONY: setup-env
 setup-env:
 	$(MAKE) .runID
@@ -74,17 +95,18 @@ destroy-env:
 	rm -fr $(VENV_DIR)
 
 .PHONY: list-platforms
-list-platforms:
+list-platforms: 
 	@docker run --rm -i -w "/workdir" -v $(PROJECT_DIR)/.ci:/workdir mikefarah/yq:4 ".PLATFORMS | keys" .e2e-platforms.yaml
 
 # Create the env file for the target platform
 .PHONY: set-env-%
 set-env-%:
-	@$(PROJECT_DIR)/.ci/scripts/yq.sh "$*"
+	@$(PROJECT_DIR)/.ci/scripts/gen-platform-env-file.py "$*"
 
 .PHONY: provision-stack
-provision-stack: .runID
+provision-stack: setup-env show-env
 	@:$(call check_defined, RUN_ID, You need to an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE) nodeUser=$(STACK_USER)" \
@@ -94,9 +116,10 @@ provision-stack: .runID
 
 .PHONY: setup-stack
 setup-stack: export TAGS = non-existing-tag
-setup-stack: .runID
+setup-stack: setup-env show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
+	source $(VENV_DIR)/bin/activate; \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeInstanceType=$(STACK_INSTANCE_TYPE) nodeUser=$(STACK_USER)" \
@@ -109,15 +132,15 @@ setup-stack: .runID
 create-stack: provision-stack setup-stack start-elastic-stack
 
 .PHONY: destroy-stack
-destroy-stack:
+destroy-stack: setup-env show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
-	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
+	source $(VENV_DIR)/bin/activate; \
+	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/teardown.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars="$(LABELS_STRING) nodeLabel=stack nodeImage=$(STACK_IMAGE) nodeUser=$(STACK_IMAGE)" \
 		--extra-vars="runId=$(RUN_ID) instanceID=$(STACK_INSTANCE_ID) nodeShellType=$(STACK_SHELL_TYPE) workspace=$(PROJECT_DIR)/ sshPublicKey=$(SSH_KEY_PUBLIC)" \
-		--ssh-common-args='$(SSH_OPTS)' \
-		-t destroy
-	rm -fr $(PROJECT_DIR)/.ci/.stack-host-ip
+		--ssh-common-args='$(SSH_OPTS)'
+	# rm -fr $(PROJECT_DIR)/.ci/.stack-host-ip
 
 .PHONY: ssh-stack
 ssh-stack:
@@ -132,9 +155,10 @@ show-stack:
 	@echo "Stack Shell         : $(STACK_SHELL_TYPE)"
 
 .PHONY: provision-node
-provision-node: .runID set-env-$(NODE_LABEL)
+provision-node: setup-env set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
@@ -143,11 +167,11 @@ provision-node: .runID set-env-$(NODE_LABEL)
 		-t provision-node
 
 .PHONY: setup-node
-setup-node: .runID set-env-$(NODE_LABEL)
+setup-node: setup-env set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, STACK_IP_ADDRESS, IP address of the stack not defined)
 	@:$(call check_defined, NODE_IP_ADDRESS, IP address of the test node not defined)
-	source $(VENV)/bin/activate; \
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/playbook.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) stackRunner=$(STACK_IP_ADDRESS) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
@@ -160,8 +184,9 @@ setup-node: .runID set-env-$(NODE_LABEL)
 create-node: provision-node setup-node
 
 .PHONY: destroy-node
-destroy-node: set-env-$(NODE_LABEL)
+destroy-node: setup-env set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/teardown.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars="$(LABELS_STRING) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeUser=$${NODE_USER}" \
@@ -170,9 +195,10 @@ destroy-node: set-env-$(NODE_LABEL)
 	rm -fr $(PROJECT_DIR)/.ci/.node-host-ip
 
 .PHONY: fetch-test-reports
-fetch-test-reports: .runID set-env-$(NODE_LABEL)
+fetch-test-reports: setup-env set-env-$(NODE_LABEL)
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
 	@:$(call check_defined, NODE_IP_ADDRESS, IP address of the test node not defined)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && $(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/fetch-test-reports.yml \
 		--private-key="$(SSH_KEY)" \
 		--extra-vars "$(LABELS_STRING) nodeLabel=$${NODE_LABEL} nodeImage=$${NODE_IMAGE} nodeInstanceType=$${NODE_INSTANCE_TYPE} nodeUser=$${NODE_USER}" \
@@ -199,8 +225,9 @@ destroy-elastic-stack:
 	ssh $(SSH_OPTS_EXTENDED) -i $(SSH_KEY) $(STACK_USER)@$(STACK_IP_ADDRESS) 'sudo docker-compose -f /root/.op/compose/profiles/fleet/docker-compose.yml down --remove-orphans'
 
 .PHONY: start-elastic-stack
-start-elastic-stack:
+start-elastic-stack: setup-env show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	PROVIDER="remote" SUITE="$(SUITE)" TAGS="non-existent-tag" \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/elastic-stack.yml \
 		--private-key="$(SSH_KEY)" \
@@ -217,8 +244,9 @@ recreate-fleet-server:
 	$(MAKE) start-elastic-stack
 
 .PHONY: run-tests
-run-tests: set-env-$(NODE_LABEL)
+run-tests: setup-env set-env-$(NODE_LABEL) show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && PROVIDER="$(PROVIDER)" SUITE="$(SUITE)" TAGS="$(TAGS)" REPORT_PREFIX="$(SUITE)_$${NODE_LABEL}_$(TAGS)" \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml \
 		--private-key="$(SSH_KEY)" \
@@ -230,8 +258,9 @@ run-tests: set-env-$(NODE_LABEL)
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && ssh $(SSH_OPTS_EXTENDED) -i $(SSH_KEY) $${NODE_USER}@$(NODE_IP_ADDRESS) "sudo bash /home/$${NODE_USER}/e2e-testing/.ci/scripts/functional-test.sh \"$(TAGS)\""
 
 .PHONY: run-tests-win
-run-tests-win: set-env-$(NODE_LABEL)
+run-tests-win: setup-env set-env-$(NODE_LABEL) show-env
 	@:$(call check_defined, RUN_ID, You need to have an unique RUN_ID. To create it please run 'make .runID' goal)
+	source $(VENV_DIR)/bin/activate; \
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && PROVIDER="$(PROVIDER)" SUITE="$(SUITE)" TAGS="$(TAGS)" REPORT_PREFIX="$(SUITE)_$${NODE_LABEL}_$(TAGS)" \
 	$(ANSIBLE_PLAYBOOK) $(PROJECT_DIR)/.ci/ansible/run-tests.yml \
 		--private-key="$(SSH_KEY)" \
@@ -241,3 +270,7 @@ run-tests-win: set-env-$(NODE_LABEL)
 		-t run-tests \
 		-i $(NODE_IP_ADDRESS),
 	. $(PROJECT_DIR)/.ci/.env-$(NODE_LABEL) && ssh $(SSH_OPTS_EXTENDED) -i $(SSH_KEY) $${NODE_USER}@$(NODE_IP_ADDRESS) "powershell \"C:/Users/$${NODE_USER}/e2e-testing/.ci/scripts/functional-test.ps1\""
+
+.PHONY: clean
+clean:
+	rm -fr "$(PROJECT_DIR)/.ci/".env-* "$(PROJECT_DIR)/.ci/.node-host-ip" "$(PROJECT_DIR)/.ci/.runID" "$(PROJECT_DIR)/.ci/.stack-host-ip" "$(PROJECT_DIR)/outputs" "$(PROJECT_DIR)/None-sshhosts" "$(PROJECT_DIR)/stack-sshhosts"