test: use filebeat step to grab Docker logs (elastic#1029)

* test: use filebeat step to grab Docker logs * fix: move filebeat step to runScript * refactor * test: chekc is correct executed * fix: typo * test: list containers * fix: do not kill containers in CI * fix: several fixes * fix: typo # Conflicts: # .ci/integrationTestDownstream.groovy # .ci/integrationTestEC.groovy # .ci/integrationTestECK.groovy # .ci/integrationTestSelector.groovy # .ci/scripts/common.sh
kuisathaverat · Feb 4, 2021 · 000176c · 000176c
1 parent 3ff3909
commit 000176c
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 29 deletions.
diff --git a/.ci/integrationTestDownstream.groovy b/.ci/integrationTestDownstream.groovy
@@ -45,6 +45,8 @@ pipeline {
     stage('Checkout'){
       options { skipDefaultCheckout() }
       steps {
+        echo "Correct PR"
+        sh('git show-ref')
         deleteDir()
         gitCheckout(basedir: "${BASE_DIR}",
           branch: "${params.INTEGRATION_TESTING_VERSION}",
@@ -102,10 +104,14 @@ pipeline {
         REUSE_CONTAINERS = "true"
       }
       steps {
-        deleteDir()
-        unstash "source"
-        dir("${BASE_DIR}"){
-          sh ".ci/scripts/all.sh"
+        withGithubNotify(context: 'All', isBlueOcean: true) {
+          deleteDir()
+          unstash "source"
+          filebeat(output: "docker-all.log", archiveOnlyOnFail: true){
+            dir("${BASE_DIR}"){
+              sh ".ci/scripts/all.sh"
+            }
+          }
         }
       }
       post {
@@ -158,11 +164,9 @@ class IntegrationTestingParallelTaskGenerator extends DefaultParallelTaskGenerat
           ]
         def label = "${tag}-${x}-${y}"
         try{
+          saveResult(x, y, 0)
           steps.runScript(label: label, agentType: tag, env: env)
           saveResult(x, y, 1)
-        } catch (e){
-          saveResult(x, y, 0)
-          steps.error("${label} tests failed : ${e.toString()}\n")
         } finally {
           steps.wrappingup(label)
         }
@@ -180,25 +184,28 @@ def runScript(Map params = [:]){
   def label = params.containsKey('label') ? params.label : params?.agentType
   def agentType = params.agentType
   def env = params.env
-  log(level: 'INFO', text: "${label}")
-  deleteDir()
-  unstash "source"
-  dir("${BASE_DIR}"){
-    withEnv(env){
-      sh """#!/bin/bash
-      export TMPDIR="${WORKSPACE}"
-      .ci/scripts/${agentType}.sh
-      """
+  def dockerLogs = label.replace(":","_").replace(";","_").replace(" ","").replace("--","-")
+  withGithubNotify(context: "${label}", isBlueOcean: true) {
+    log(level: 'INFO', text: "${label}")
+    deleteDir()
+    unstash "source"
+    filebeat(output: "docker-${dockerLogs}.log", archiveOnlyOnFail: true){
+      sh 'docker ps -a'
+      dir("${BASE_DIR}"){
+        withEnv(env){
+          sh(label: "Testing ${agentType}", script: ".ci/scripts/${agentType}.sh")
+          sh 'docker ps -a'
+        }
+      }
     }
   }
 }
 
 def wrappingup(label){
   dir("${BASE_DIR}"){
-    def stepName = label.replace(";","/")
-      .replace("--","_")
-      .replace(".","_")
-    sh("./scripts/docker-get-logs.sh '${stepName}'|| echo 0")
+    def testResultsFolder = 'tests/results'
+    def testResultsPattern = "${testResultsFolder}/*-junit*.xml"
+    def labelFolder = normalise(label)
     sh('make stop-env || echo 0')
     archiveArtifacts(
         allowEmptyArchive: true,
@@ -210,3 +217,7 @@ def wrappingup(label){
       testResults: "**/tests/results/*-junit*.xml")
   }
 }
+
+def normalise(label) {
+  return label?.replace(';','/').replace('--','_').replace('.','_').replace(' ','_')
+}
diff --git a/.ci/scripts/common.sh b/.ci/scripts/common.sh
@@ -11,8 +11,9 @@ function runTests() {
     trap "stopEnv" EXIT
     targets="destroy-env"
   fi
-  targets="${targets} $@"
+  targets="${targets} $*"
   export VENV=${VENV:-${TMPDIR:-/tmp/}venv-$$}
+  # shellcheck disable=SC2086
   make ${targets}
 }
 
@@ -27,19 +28,13 @@ if [ -n "${APM_SERVER_BRANCH}" ]; then
  fi
 fi
 
-if [ -z "${DISABLE_BUILD_PARALLEL}" ]; then
+if [ -z "${DISABLE_BUILD_PARALLEL}" ] || [ "${DISABLE_BUILD_PARALLEL}" = "false" ]; then
  BUILD_OPTS="${BUILD_OPTS} --build-parallel"
 fi
 
 ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION:-'6.8 --release'}
 
-# assume we're under CI if BUILD_NUMBER is set
-if [ -n "${BUILD_NUMBER}" ]; then
-  # kill any running containers under CI
-  [ -n "$(docker ps -aq)" ] && docker ps -aq | xargs -t docker rm -f || true
-fi
-
 echo "ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}"
 echo "APM_SERVER_BRANCH_VERSION=${APM_SERVER_BRANCH_VERSION}"
 echo "APM_SERVER_BRANCH_TYPE=${APM_SERVER_BRANCH_TYPE}"
-echo "BUILD_OPTS=${BUILD_OPTS}"
+echo "BUILD_OPTS=${BUILD_OPTS}"