ci: refactor travis to use similar approach in jenkins (#588)

elastic · Jul 19, 2019 · e92cea0 · e92cea0
1 parent 1256237
commit e92cea0
Show file tree

Hide file tree

Showing 16 changed files with 223 additions and 289 deletions.
diff --git a/.jenkins-edge.yml b/.jenkins-edge.yml
@@ -0,0 +1,2 @@
+GO_VERSION:
+  - "master"
diff --git a/.jenkins.yml b/.jenkins.yml
@@ -1,2 +1,8 @@
 GO_VERSION:
-  - "1.12.0"
+  - stable
+  - "1.x"
+  - "1.8.x"
+  - "1.9.x"
+  - "1.10.x"
+  - "1.11.x"
+  - "1.12.x"
diff --git a/.travis.yml b/.travis.yml
@@ -23,60 +23,19 @@ matrix:
     - go: master
 
 before_install:
-  - |
-      if (go run scripts/mingoversion.go 1.10 &>/dev/null); then
-        go get -v golang.org/x/lint/golint;
-        go get -v golang.org/x/tools/cmd/goimports;
-        go get -v github.com/elastic/go-licenser;
-      fi
-  - |
-      if (! go run scripts/mingoversion.go 1.10 &>/dev/null); then
-        # Before Go 1.10.0, pin gin-gonic to v1.3.0.
-        mkdir -p $GOPATH/src/github.com/gin-gonic;
-        (
-          cd $GOPATH/src/github.com/gin-gonic &&
-          git clone https://github.com/gin-gonic/gin &&
-          cd gin && git checkout v1.3.0
-        );
-      fi
-  - |
-      if (! go run scripts/mingoversion.go 1.9 &>/dev/null); then
-        # For Go 1.8.x, pin go-sql-driver to v1.4.1,
-        # the last release that supports Go 1.8.
-        mkdir -p $GOPATH/src/github.com/go-sql-driver;
-        (
-          cd $GOPATH/src/github.com/go-sql-driver &&
-          git clone https://github.com/go-sql-driver/mysql &&
-          cd mysql && git checkout v1.4.1
-        );
-        # Pin olivere/elastic to release-branch.v6 for Go 1.8.
-        mkdir -p $GOPATH/src/github.com/olivere/elastic;
-        (
-          cd $GOPATH/src/github.com/olivere &&
-          git clone https://github.com/olivere/elastic &&
-          cd elastic && git checkout release-branch.v6
-        );
-      fi;
+  - ./scripts/before_install.sh
 
 script:
   - make install check
 
 jobs:
   include:
-    - os: osx     
-
-    - name: "W3C Distributed Tracing Validation"
-      sudo: required
-      services:
-        - docker
-      script:
-        - scripts/docker-compose-testing run -T --rm trace-context-harness
+    - os: osx
 
     - stage: coverage
       sudo: required
       services:
         - docker
       script:
-        - scripts/docker-compose-testing up -d --build
-        - scripts/docker-compose-testing run -T --rm go-agent-tests make coverage > coverage.txt
+        - ./scripts/docker-test.sh
         - bash <(curl -s https://codecov.io/bash)
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -29,7 +29,7 @@ pipeline {
     issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
   }
   parameters {
-    string(name: 'GO_VERSION', defaultValue: "1.12.0", description: "Go version to use.")
+    string(name: 'GO_VERSION', defaultValue: "1.12.7", description: "Go version to use.")
     booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
     booleanParam(name: 'test_ci', defaultValue: true, description: 'Enable test')
     booleanParam(name: 'docker_test_ci', defaultValue: true, description: 'Enable run docker tests')
@@ -41,10 +41,10 @@ pipeline {
       agent { label 'linux && immutable' }
       options { skipDefaultCheckout() }
       environment {
-        PATH = "${env.PATH}:${env.WORKSPACE}/bin"
         HOME = "${env.WORKSPACE}"
         GOPATH = "${env.WORKSPACE}"
         GO_VERSION = "${params.GO_VERSION}"
+        PATH = "${env.PATH}:${env.WORKSPACE}/bin"
       }
       stages {
         /**
@@ -57,69 +57,69 @@ pipeline {
           }
         }
         /**
-        Build on a linux environment.
+        Execute unit tests.
         */
-        stage('build') {
+        stage('Tests') {
+          agent { label 'linux && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression { return params.test_ci }
+          }
           steps {
-            withGithubNotify(context: 'Build') {
+            withGithubNotify(context: 'Tests', tab: 'tests') {
               deleteDir()
               unstash 'source'
               dir("${BASE_DIR}"){
-                sh './scripts/jenkins/build.sh'
+                script {
+                  def go = readYaml(file: '.jenkins.yml')
+                  def parallelTasks = [:]
+                  go['GO_VERSION'].each{ version ->
+                    parallelTasks["Go-${version}"] = generateStep(version)
+                  }
+                  // For the cutting edge
+                  def edge = readYaml(file: '.jenkins-edge.yml')
+                  edge['GO_VERSION'].each{ version ->
+                    parallelTasks["Go-${version}"] = generateStepAndCatchError(version)
+                  }
+                  parallel(parallelTasks)
+                }
               }
             }
           }
         }
-      }
-    }
-    stage('Test') {
-      failFast true
-      parallel {
-        /**
-          Run unit tests and store the results in Jenkins.
-        */
-        stage('Unit Test') {
+        stage('Coverage') {
           agent { label 'linux && immutable' }
           options { skipDefaultCheckout() }
-          environment {
-            PATH = "${env.PATH}:${env.WORKSPACE}/bin"
-            HOME = "${env.WORKSPACE}"
-            GOPATH = "${env.WORKSPACE}"
-            GO_VERSION = "${params.GO_VERSION}"
-          }
           when {
             beforeAgent true
-            expression { return params.test_ci }
+            expression { return params.docker_test_ci }
           }
           steps {
-            withGithubNotify(context: 'Unit Test', tab: 'tests') {
+            withGithubNotify(context: 'Coverage') {
               deleteDir()
               unstash 'source'
               dir("${BASE_DIR}"){
-                sh './scripts/jenkins/test.sh'
+                sh script: './scripts/jenkins/before_install.sh', label: 'Install dependencies'
+                sh script: './scripts/jenkins/docker-test.sh', label: 'Docker tests'
               }
             }
           }
           post {
             always {
+              coverageReport("${BASE_DIR}/build/coverage")
+              codecov(repo: env.REPO, basedir: "${BASE_DIR}",
+                flags: "-f build/coverage/coverage.cov -X search",
+                secret: "${CODECOV_SECRET}")
               junit(allowEmptyResults: true,
                 keepLongStdio: true,
                 testResults: "${BASE_DIR}/build/junit-*.xml")
             }
           }
         }
-        /**
-          Run Benchmarks and send the results to ES.
-        */
-        stage('Benchmarks') {
+        stage('Benchmark') {
           agent { label 'linux && immutable' }
           options { skipDefaultCheckout() }
-          environment {
-            PATH = "${env.PATH}:${env.WORKSPACE}/bin"
-            HOME = "${env.WORKSPACE}"
-            GOPATH = "${env.WORKSPACE}"
-            GO_VERSION = "${params.GO_VERSION}"
-          }
           when {
             beforeAgent true
             allOf {
@@ -134,62 +134,59 @@ pipeline {
             }
           }
           steps {
-            withGithubNotify(context: 'Benchmarks', tab: 'tests') {
+            withGithubNotify(context: 'Benchmark', tab: 'tests') {
               deleteDir()
               unstash 'source'
               dir("${BASE_DIR}"){
-                sh './scripts/jenkins/bench.sh'
-                sendBenchmarks(file: 'build/bench.out', index: "benchmark-go")
+                sh script: './scripts/jenkins/before_install.sh', label: 'Install dependencies'
+                sh script: './scripts/jenkins/bench.sh', label: 'Benchmarking'
+                sendBenchmarks(file: 'build/bench.out', index: 'benchmark-go')
               }
             }
           }
-          post {
-            always {
-              junit(allowEmptyResults: true,
-                keepLongStdio: true,
-                testResults: "${BASE_DIR}/build/junit-*.xml")
-            }
-          }
         }
-        /**
-          Run tests in a docker container and store the results in jenkins and codecov.
-        */
-        stage('Docker Tests') {
-          agent { label 'linux && docker && immutable' }
-          options { skipDefaultCheckout() }
-          environment {
-            PATH = "${env.PATH}:${env.WORKSPACE}/bin"
-            HOME = "${env.WORKSPACE}"
-            GOPATH = "${env.WORKSPACE}"
-            GO_VERSION = "${params.GO_VERSION}"
-          }
-          when {
-            beforeAgent true
-            expression { return params.docker_test_ci }
-          }
-          steps {
-            withGithubNotify(context: 'Docker Tests', tab: 'tests') {
-              deleteDir()
-              unstash 'source'
-              dir("${BASE_DIR}"){
-                sh './scripts/jenkins/docker-test.sh'
-              }
-            }
+      }
+    }
+    stage('Windows') {
+      agent { label 'windows' }
+      options { skipDefaultCheckout() }
+      environment {
+        GOROOT = "c:\\Go"
+        GOPATH = "${env.WORKSPACE}"
+        PATH = "${env.PATH};${env.GOROOT}\\bin;${env.GOPATH}\\bin"
+        GO_VERSION = "${params.GO_VERSION}"
+      }
+      steps {
+        withGithubNotify(context: 'Build-Test - Windows') {
+          cleanDir("${WORKSPACE}/${BASE_DIR}")
+          unstash 'source'
+          dir("${BASE_DIR}"){
+            bat script: 'scripts/jenkins/windows/install-tools.bat', label: 'Install tools'
+            bat script: 'scripts/jenkins/windows/build-test.bat', label: 'Build and test'
           }
-          post {
-            always {
-              coverageReport("${BASE_DIR}/build/coverage")
-              junit(allowEmptyResults: true,
-                keepLongStdio: true,
-                testResults: "${BASE_DIR}/build/junit-*.xml")
-              codecov(repo: env.REPO, basedir: "${BASE_DIR}",
-                flags: "-f build/coverage/coverage.cov -X search",
-                secret: "${CODECOV_SECRET}")
-            }
+        }
+      }
+      post {
+        always {
+          junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/junit-*.xml")
+          dir("${BASE_DIR}"){
+            bat script: 'scripts/jenkins/windows/uninstall-tools.bat', label: 'Uninstall tools'
           }
+          cleanWs(disableDeferredWipeout: true, notFailBuild: true)
         }
       }
     }
+    stage('OSX') {
+      agent none
+      /** TODO: As soon as MacOSX are available we will provide the stage implementation */
+      when {
+        beforeAgent true
+        expression { return false }
+      }
+      steps {
+        echo 'TBD'
+      }
+    }
     /**
       Build the documentation.
     */
@@ -246,3 +243,41 @@ pipeline {
     }
   }
 }
+
+def generateStep(version){
+  return {
+    node('docker && linux && immutable'){
+      try {
+        deleteDir()
+        unstash 'source'
+        echo "${version}"
+        dir("${BASE_DIR}"){
+          withEnv([
+            "GO_VERSION=${version}",
+            "HOME=${WORKSPACE}"]) {
+            sh script: './scripts/jenkins/before_install.sh', label: 'Install dependencies'
+            sh script: './scripts/jenkins/build-test.sh', label: 'Build and test'
+          }
+        }
+      } catch(e){
+        error(e.toString())
+      } finally {
+        junit(allowEmptyResults: true,
+          keepLongStdio: true,
+          testResults: "${BASE_DIR}/build/junit-*.xml")
+      }
+    }
+  }
+}
+
+def generateStepAndCatchError(version){
+  return {
+    catchError(buildResult: 'SUCCESS', message: 'Cutting Edge Tests', stageResult: 'UNSTABLE') {
+      generateStep(version)
+    }
+  }
+}
+
+def cleanDir(path){
+  powershell label: "Clean ${path}", script: "Remove-Item -Recurse -Force ${path}"
+}
diff --git a/scripts/before_install.sh b/scripts/before_install.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+if (go run scripts/mingoversion.go 1.10 &>/dev/null); then
+  go get -v golang.org/x/lint/golint;
+  go get -v golang.org/x/tools/cmd/goimports;
+  go get -v github.com/elastic/go-licenser;
+fi
+
+if (! go run scripts/mingoversion.go 1.10 &>/dev/null); then
+  # Before Go 1.10.0, pin gin-gonic to v1.3.0.
+  mkdir -p "${GOPATH}/src/github.com/gin-gonic";
+  (
+    cd "${GOPATH}/src/github.com/gin-gonic" &&
+    git clone https://github.com/gin-gonic/gin &&
+    cd gin && git checkout v1.3.0
+  );
+fi
+
+if (! go run scripts/mingoversion.go 1.9 &>/dev/null); then
+  # For Go 1.8.x, pin go-sql-driver to v1.4.1,
+  # the last release that supports Go 1.8.
+  mkdir -p "${GOPATH}/src/github.com/go-sql-driver";
+  (
+    cd "${GOPATH}/src/github.com/go-sql-driver" &&
+    git clone https://github.com/go-sql-driver/mysql &&
+    cd mysql && git checkout v1.4.1
+  );
+  # Pin olivere/elastic to release-branch.v6 for Go 1.8.
+  mkdir -p "${GOPATH}/src/github.com/olivere/elastic";
+  (
+    cd "${GOPATH}/src/github.com/olivere" &&
+    git clone https://github.com/olivere/elastic &&
+    cd elastic && git checkout release-branch.v6
+  );
+fi