test: use filebeat step to grab Docker logs (#1029)

* test: use filebeat step to grab Docker logs * fix: move filebeat step to runScript * refactor * test: chekc is correct executed * fix: typo * test: list containers * fix: do not kill containers in CI * fix: several fixes * fix: typo
elastic · Feb 4, 2021 · 951e5ab · 951e5ab
1 parent e3d3035
commit 951e5ab
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 40 deletions.
diff --git a/.ci/integrationTestDownstream.groovy b/.ci/integrationTestDownstream.groovy
@@ -49,6 +49,8 @@ pipeline {
     stage('Checkout'){
       options { skipDefaultCheckout() }
       steps {
+        echo "Correct PR"
+        sh('git show-ref')
         deleteDir()
         gitCheckout(basedir: "${BASE_DIR}",
           branch: "${params.INTEGRATION_TESTING_VERSION}",
@@ -115,8 +117,10 @@ pipeline {
         withGithubNotify(context: 'All', isBlueOcean: true) {
           deleteDir()
           unstash "source"
-          dir("${BASE_DIR}"){
-            sh ".ci/scripts/all.sh"
+          filebeat(output: "docker-all.log", archiveOnlyOnFail: true){
+            dir("${BASE_DIR}"){
+              sh ".ci/scripts/all.sh"
+            }
           }
         }
       }
@@ -190,11 +194,9 @@ class IntegrationTestingParallelTaskGenerator extends DefaultParallelTaskGenerat
           ]
         def label = "${tag}-${x}-${y}"
         try{
+          saveResult(x, y, 0)
           steps.runScript(label: label, agentType: tag, env: env)
           saveResult(x, y, 1)
-        } catch (e){
-          saveResult(x, y, 0)
-          steps.error("${label} tests failed : ${e.toString()}\n")
         } finally {
           steps.wrappingup(label)
         }
@@ -212,13 +214,18 @@ def runScript(Map params = [:]){
   def label = params.containsKey('label') ? params.label : params?.agentType
   def agentType = params.agentType
   def env = params.env
+  def dockerLogs = label.replace(":","_").replace(";","_").replace(" ","").replace("--","-")
   withGithubNotify(context: "${label}", isBlueOcean: true) {
     log(level: 'INFO', text: "${label}")
     deleteDir()
     unstash "source"
-    dir("${BASE_DIR}"){
-      withEnv(env){
-        sh(label: "Testing ${agentType}", script: ".ci/scripts/${agentType}.sh")
+    filebeat(output: "docker-${dockerLogs}.log", archiveOnlyOnFail: true){
+      sh 'docker ps -a'
+      dir("${BASE_DIR}"){
+        withEnv(env){
+          sh(label: "Testing ${agentType}", script: ".ci/scripts/${agentType}.sh")
+          sh 'docker ps -a'
+        }
       }
     }
   }
@@ -229,9 +236,6 @@ def wrappingup(label){
     def testResultsFolder = 'tests/results'
     def testResultsPattern = "${testResultsFolder}/*-junit*.xml"
     def labelFolder = normalise(label)
-    if(currentBuild.result == 'FAILURE' || currentBuild.result == 'UNSTABLE'){
-      dockerLogs(step: label, failNever: true)
-    }
     sh('make stop-env || echo 0')
     sh(label: 'Folder to aggregate test results from stages',
        script: "mkdir -p ${labelFolder}/${testResultsFolder} && cp -rf ${testResultsPattern} ${labelFolder}/${testResultsFolder}")

diff --git a/.ci/integrationTestEC.groovy b/.ci/integrationTestEC.groovy
@@ -199,8 +199,10 @@ def runTest(test){
   unstash 'source'
   withConfigEnv(){
     catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-      dir("${BASE_DIR}"){
-        sh ".ci/scripts/${test}.sh"
+      filebeat(output: "docker-${ELASTIC_STACK_VERSION}-${test}.log", archiveOnlyOnFail: true){
+        dir("${BASE_DIR}"){
+          sh ".ci/scripts/${test}.sh"
+        }
       }
     }
   }
@@ -262,10 +264,6 @@ def withConfigEnv(Closure body) {
 def grabResultsAndLogs(label){
   withConfigEnv(){
     dir("${BASE_DIR}"){
-      if(currentBuild.result == 'FAILURE' || currentBuild.result == 'UNSTABLE'){
-        dockerLogs(step: label, failNever: true)
-        sh('.ci/scripts/remove_env.sh docker-info')
-      }
       sh('make stop-env || echo 0')
       archiveArtifacts(
           allowEmptyArchive: true,

diff --git a/.ci/integrationTestECK.groovy b/.ci/integrationTestECK.groovy
@@ -215,8 +215,10 @@ def runTest(test){
   unstash 'source'
   withConfigEnv(){
     catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-      dir("${BASE_DIR}"){
-        sh ".ci/scripts/${test}.sh"
+      filebeat(output: "docker-${ELASTIC_STACK_VERSION}-${test}.log", archiveOnlyOnFail: true){
+        dir("${BASE_DIR}"){
+          sh ".ci/scripts/${test}.sh"
+        }
       }
     }
   }

diff --git a/.ci/integrationTestSelector.groovy b/.ci/integrationTestSelector.groovy
@@ -76,8 +76,10 @@ pipeline {
           steps {
             deleteDir()
             unstash "source"
-            dir("${BASE_DIR}"){
-              sh(label: "Testing ${NAME} ${APP}", script: ".ci/scripts/agent.sh ${NAME} ${APP}")
+            filebeat(output: "docker-${NAME}-${APP}.log", archiveOnlyOnFail: true){
+              dir("${BASE_DIR}"){
+                sh(label: "Testing ${NAME} ${APP}", script: ".ci/scripts/agent.sh ${NAME} ${APP}")
+              }
             }
           }
           post {
@@ -97,8 +99,10 @@ pipeline {
           steps {
             deleteDir()
             unstash "source"
-            dir("${BASE_DIR}"){
-              sh(label: "Testing ${NAME} ${OPBEANS_APP}", script: ".ci/scripts/opbeans-app.sh ${NAME} ${APP} ${OPBEANS_APP}")
+            filebeat(output: "docker-${OPBEANS_APP}.log", archiveOnlyOnFail: true){
+              dir("${BASE_DIR}"){
+                sh(label: "Testing ${NAME} ${OPBEANS_APP}", script: ".ci/scripts/opbeans-app.sh ${NAME} ${APP} ${OPBEANS_APP}")
+              }
             }
           }
           post {
@@ -122,8 +126,10 @@ pipeline {
               sh script: ".ci/bump-version.sh ${env.BUILD_OPTS.replaceAll('--rum-agent-branch ', '')} false", label: 'Bump version'
               sh script: 'make build', label: 'Build docker image with the new rum agent'
             }
-            dir("${BASE_DIR}"){
-              sh(label: 'Testing RUM', script: '.ci/scripts/opbeans-rum.sh')
+            filebeat(output: "docker-opbeans-rum.log", archiveOnlyOnFail: true){
+              dir("${BASE_DIR}"){
+                sh(label: 'Testing RUM', script: '.ci/scripts/opbeans-rum.sh')
+              }
             }
           }
           post {
@@ -147,8 +153,10 @@ pipeline {
       steps {
         deleteDir()
         unstash "source"
-        dir("${BASE_DIR}"){
-          sh ".ci/scripts/all.sh"
+        filebeat(output: "docker-all.log", archiveOnlyOnFail: true){
+          dir("${BASE_DIR}"){
+            sh ".ci/scripts/all.sh"
+          }
         }
       }
       post {
@@ -196,8 +204,10 @@ pipeline {
       steps {
         deleteDir()
         unstash "source"
-        dir("${BASE_DIR}"){
-          sh ".ci/scripts/opbeans.sh"
+        filebeat(output: "docker-opbeans.log", archiveOnlyOnFail: true){
+          dir("${BASE_DIR}"){
+            sh ".ci/scripts/opbeans.sh"
+          }
         }
       }
       post {
@@ -218,9 +228,6 @@ pipeline {
 def wrappingup(Map params = [:]){
   def isJunit = params.containsKey('isJunit') ? params.get('isJunit') : true
   dir("${BASE_DIR}"){
-    if(currentBuild.result == 'FAILURE' || currentBuild.result == 'UNSTABLE'){
-      dockerLogs(step: "${env.NAME}", failNever: true)
-    }
     sh('make stop-env || echo 0')
     def testResultsPattern = 'tests/results/*-junit*.xml'
     archiveArtifacts(

diff --git a/.ci/scripts/common.sh b/.ci/scripts/common.sh
@@ -11,8 +11,9 @@ function runTests() {
     trap "stopEnv" EXIT
     targets="destroy-env"
   fi
-  targets="${targets} $@"
+  targets="${targets} $*"
   export VENV=${VENV:-${TMPDIR:-/tmp/}venv-$$}
+  # shellcheck disable=SC2086
   make ${targets}
 }
 
@@ -27,18 +28,12 @@ if [ -n "${APM_SERVER_BRANCH}" ]; then
  fi
 fi
 
-if [ -z "${DISABLE_BUILD_PARALLEL}" -o "${DISABLE_BUILD_PARALLEL}" = "false" ]; then
+if [ -z "${DISABLE_BUILD_PARALLEL}" ] || [ "${DISABLE_BUILD_PARALLEL}" = "false" ]; then
  BUILD_OPTS="${BUILD_OPTS} --build-parallel"
 fi
 
 ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION:-'7.0.0'}
 
-# assume we're under CI if BUILD_NUMBER is set
-if [ -n "${BUILD_NUMBER}" ]; then
-  # kill any running containers under CI
-  [ -n "$(docker ps -aq)" ] && docker ps -aq | xargs -t docker rm -f || true
-fi
-
 echo "ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}"
 echo "APM_SERVER_BRANCH_VERSION=${APM_SERVER_BRANCH_VERSION}"
 echo "APM_SERVER_BRANCH_TYPE=${APM_SERVER_BRANCH_TYPE}"