Jenkins pipeline with parallel cigroups (#45285)

* Pipeline

* WIP some work for parallelization with ciGroups

* Fix xpack kibana install dir, and add some debugging

* Attempt to quick fix a few tests

* Revert "Revert "Revert "[ci] compress jobs for CI stability" (#44584)""

This reverts commit 078ac28.

* Recombine test groups, and try runbld again

* Mostly cleanup, and fix failed_tests reporting to hopefully work for both pipeline and non-pipeline

* Fix typo in shell script

* Remove some debug code

* Add support for changing es transport.port during testing via TEST_ES_TRANSPORT_PORT

* Fix test that uses hard-coded es transport port and add it back in to parallel groups

* Disable checks reporter again for now

* Set env var for TEST_ES_TRANSPORT_PORT in pipeline

* Update Jenkinsfile for shorter testrunner labels

* Fix another hard-coded transport port

* Fix a new test with hard-coded URLs

* Jenkinsfile cleanup and fix one of the groups

* Fix double slash

* Testing vault credentials on jenkins server

* Add a non-existent credential

* Revert "Add a non-existent credential"

This reverts commit 0dc234c.

* Try github-checks-reporter again

* github-checks-reporter should only run for elastic/kibana, forks won't work

* Clean up some debug code

* Changing names around to try to make BlueOcean UI a little better

* Add more stages

* Make some changes to stage structure to mirror a nested example from CloudBees

* Handle TODOs, and some cleanup in Jenkinsfile

* Pass GIT_BRANCH when started without GHPRB, fix branch check

* Fix mailer problem and add code that ensures all tests are in cigroups back in

* Test adding worker/job name to junit report paths

* Remove some duplication from ci_setup scripts

* Fix unit test that uses junit path

* Don't reinstall node every time setup_env is run

* Fix yarn install logic

* Fix another unit test that uses junit output dir

* Download latest ES snapshot after kibana builds

* Make sure junit reports are always processed

* Add two failing tests for testing purposes

* Add support to Jenkinsfile for kibana build e-mails

* Remove some debug code for email sending

* Change JOB env handling in junit paths and move it to a sub-directory

* Revert "Add two failing tests for testing purposes"

This reverts commit 5715203e26922a93483feb0ebb8bb3fdcc3daf8c.

* Fix junit report path in test

* Don't send kibana emails on build abort

* Address PR feedback, formatting and use built-in url formatting library

* Fix path formatting for functional test

* Add email sending back in to Jenkinsfile

* Fix another unit test with path problem

.ci/jobs.yml

@@ -1,21 +1,34 @@
 JOB:
-  - intake
-  - firefoxSmoke
+  - kibana-intake
+  - x-pack-intake
+  - kibana-firefoxSmoke
   - kibana-ciGroup1
   - kibana-ciGroup2
   - kibana-ciGroup3
   - kibana-ciGroup4
   - kibana-ciGroup5
   - kibana-ciGroup6
-  # - kibana-visualRegression
+  - kibana-ciGroup7
+  - kibana-ciGroup8
+  - kibana-ciGroup9
+  - kibana-ciGroup10
+  - kibana-ciGroup11
+  - kibana-ciGroup12
+  - kibana-visualRegression
 
   # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
+  - x-pack-firefoxSmoke
   - x-pack-ciGroup1
   - x-pack-ciGroup2
   - x-pack-ciGroup3
   - x-pack-ciGroup4
   - x-pack-ciGroup5
-  # - x-pack-visualRegression
+  - x-pack-ciGroup6
+  - x-pack-ciGroup7
+  - x-pack-ciGroup8
+  - x-pack-ciGroup9
+  - x-pack-ciGroup10
+  - x-pack-visualRegression
 
 # `~` is yaml for `null`
 exclude: ~

.ci/run.sh

@@ -11,7 +11,7 @@ source src/dev/ci_setup/setup.sh
 source src/dev/ci_setup/checkout_sibling_es.sh
 
 case "$JOB" in
-intake)
+kibana-intake)
   ./test/scripts/jenkins_unit.sh
   ;;
 kibana-ciGroup*)
@@ -21,16 +21,22 @@ kibana-ciGroup*)
 kibana-visualRegression*)
   ./test/scripts/jenkins_visual_regression.sh
   ;;
-firefoxSmoke*)
+kibana-firefoxSmoke*)
   ./test/scripts/jenkins_firefox_smoke.sh
   ;;
+x-pack-intake)
+  ./test/scripts/jenkins_xpack.sh
+  ;;
 x-pack-ciGroup*)
   export CI_GROUP="${JOB##x-pack-ciGroup}"
   ./test/scripts/jenkins_xpack_ci_group.sh
   ;;
 x-pack-visualRegression*)
   ./test/scripts/jenkins_xpack_visual_regression.sh
   ;;
+x-pack-firefoxSmoke*)
+  ./test/scripts/jenkins_xpack_firefox_smoke.sh
+  ;;
 *)
   echo "JOB '$JOB' is not implemented."
   exit 1

Jenkinsfile

@@ -0,0 +1,268 @@
+#!/bin/groovy
+
+properties([
+  durabilityHint('PERFORMANCE_OPTIMIZED'),
+])
+
+stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
+  timeout(time: 180, unit: 'MINUTES') {
+    timestamps {
+      ansiColor('xterm') {
+        catchError {
+          parallel([
+            'kibana-intake-agent': legacyJobRunner('kibana-intake'),
+            'x-pack-intake-agent': legacyJobRunner('x-pack-intake'),
+            'kibana-oss-agent': withWorkers('kibana-oss-tests', { buildOss() }, [
+              'oss-ciGroup1': getOssCiGroupWorker(1),
+              'oss-ciGroup2': getOssCiGroupWorker(2),
+              'oss-ciGroup3': getOssCiGroupWorker(3),
+              'oss-ciGroup4': getOssCiGroupWorker(4),
+              'oss-ciGroup5': getOssCiGroupWorker(5),
+              'oss-ciGroup6': getOssCiGroupWorker(6),
+              'oss-ciGroup7': getOssCiGroupWorker(7),
+              'oss-ciGroup8': getOssCiGroupWorker(8),
+              'oss-ciGroup9': getOssCiGroupWorker(9),
+              'oss-ciGroup10': getOssCiGroupWorker(10),
+              'oss-ciGroup11': getOssCiGroupWorker(11),
+              'oss-ciGroup12': getOssCiGroupWorker(12),
+              'oss-visualRegression': getPostBuildWorker('visualRegression', { runbld './test/scripts/jenkins_visual_regression.sh' }),
+              'oss-firefoxSmoke': getPostBuildWorker('firefoxSmoke', { runbld './test/scripts/jenkins_firefox_smoke.sh' }),
+            ]),
+            'kibana-xpack-agent': withWorkers('kibana-xpack-tests', { buildXpack() }, [
+              'xpack-ciGroup1': getXpackCiGroupWorker(1),
+              'xpack-ciGroup2': getXpackCiGroupWorker(2),
+              'xpack-ciGroup3': getXpackCiGroupWorker(3),
+              'xpack-ciGroup4': getXpackCiGroupWorker(4),
+              'xpack-ciGroup5': getXpackCiGroupWorker(5),
+              'xpack-ciGroup6': getXpackCiGroupWorker(6),
+              'xpack-ciGroup7': getXpackCiGroupWorker(7),
+              'xpack-ciGroup8': getXpackCiGroupWorker(8),
+              'xpack-ciGroup9': getXpackCiGroupWorker(9),
+              'xpack-ciGroup10': getXpackCiGroupWorker(10),
+              'xpack-firefoxSmoke': getPostBuildWorker('xpack-firefoxSmoke', { runbld './test/scripts/jenkins_xpack_firefox_smoke.sh' }),
+              'xpack-visualRegression': getPostBuildWorker('xpack-visualRegression', { runbld './test/scripts/jenkins_xpack_visual_regression.sh' }),
+            ]),
+          ])
+        }
+        node('flyweight') {
+          sendMail()
+        }
+      }
+    }
+  }
+}
+
+def withWorkers(name, preWorkerClosure = {}, workerClosures = [:]) {
+  return {
+    jobRunner('tests-xl') {
+      try {
+        doSetup()
+        preWorkerClosure()
+
+        def nextWorker = 1
+        def worker = { workerClosure ->
+          def workerNumber = nextWorker
+          nextWorker++
+
+          return {
+            workerClosure(workerNumber)
+          }
+        }
+
+        def workers = [:]
+        workerClosures.each { workerName, workerClosure ->
+          workers[workerName] = worker(workerClosure)
+        }
+
+        parallel(workers)
+      } finally {
+        catchError {
+          uploadAllGcsArtifacts(name)
+        }
+
+        catchError {
+          publishJunit()
+        }
+      }
+    }
+  }
+}
+
+def getPostBuildWorker(name, closure) {
+  return { workerNumber ->
+    def kibanaPort = "61${workerNumber}1"
+    def esPort = "61${workerNumber}2"
+    def esTransportPort = "61${workerNumber}3"
+
+    withEnv([
+      "CI_WORKER_NUMBER=${workerNumber}",
+      "TEST_KIBANA_HOST=localhost",
+      "TEST_KIBANA_PORT=${kibanaPort}",
+      "TEST_KIBANA_URL=http://elastic:changeme@localhost:${kibanaPort}",
+      "TEST_ES_URL=http://elastic:changeme@localhost:${esPort}",
+      "TEST_ES_TRANSPORT_PORT=${esTransportPort}",
+      "IS_PIPELINE_JOB=1",
+    ]) {
+      closure()
+    }
+  }
+}
+
+def getOssCiGroupWorker(ciGroup) {
+  return getPostBuildWorker("ciGroup" + ciGroup, {
+    withEnv([
+      "CI_GROUP=${ciGroup}",
+      "JOB=kibana-ciGroup${ciGroup}",
+    ]) {
+      runbld "./test/scripts/jenkins_ci_group.sh"
+    }
+  })
+}
+
+def getXpackCiGroupWorker(ciGroup) {
+  return getPostBuildWorker("xpack-ciGroup" + ciGroup, {
+    withEnv([
+      "CI_GROUP=${ciGroup}",
+      "JOB=xpack-kibana-ciGroup${ciGroup}",
+    ]) {
+      runbld "./test/scripts/jenkins_xpack_ci_group.sh"
+    }
+  })
+}
+
+def legacyJobRunner(name) {
+  return {
+    parallel([
+      "${name}": {
+        withEnv([
+          "JOB=${name}",
+        ]) {
+          jobRunner('linux && immutable') {
+            try {
+              runbld '.ci/run.sh'
+            } finally {
+              catchError {
+                uploadAllGcsArtifacts(name)
+              }
+              catchError {
+                publishJunit()
+              }
+            }
+          }
+        }
+      }
+    ])
+  }
+}
+
+def jobRunner(label, closure) {
+  node(label) {
+    def scmVars = checkout scm
+
+    withEnv([
+      "CI=true",
+      "HOME=${env.JENKINS_HOME}",
+      "PR_SOURCE_BRANCH=${env.ghprbSourceBranch}",
+      "PR_TARGET_BRANCH=${env.ghprbTargetBranch}",
+      "PR_AUTHOR=${env.ghprbPullAuthorLogin}",
+      "TEST_BROWSER_HEADLESS=1",
+      "GIT_BRANCH=${scmVars.GIT_BRANCH}",
+    ]) {
+      withCredentials([
+        string(credentialsId: 'vault-addr', variable: 'VAULT_ADDR'),
+        string(credentialsId: 'vault-role-id', variable: 'VAULT_ROLE_ID'),
+        string(credentialsId: 'vault-secret-id', variable: 'VAULT_SECRET_ID'),
+      ]) {
+        // scm is configured to check out to the ./kibana directory
+        dir('kibana') {
+          closure()
+        }
+      }
+    }
+  }
+}
+
+// TODO what should happen if GCS, Junit, or email publishing fails? Unstable build? Failed build?
+
+def uploadGcsArtifact(workerName, pattern) {
+  def storageLocation = "gs://kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" // TODO
+  // def storageLocation = "gs://kibana-pipeline-testing/jobs/pipeline-test/${BUILD_NUMBER}/${workerName}"
+
+  googleStorageUpload(
+    credentialsId: 'kibana-ci-gcs-plugin',
+    bucket: storageLocation,
+    pattern: pattern,
+    sharedPublicly: true,
+    showInline: true,
+  )
+}
+
+def uploadAllGcsArtifacts(workerName) {
+  def ARTIFACT_PATTERNS = [
+    'target/kibana-*',
+    'target/junit/**/*',
+    'test/**/screenshots/**/*.png',
+    'test/functional/failure_debug/html/*.html',
+    'x-pack/test/**/screenshots/**/*.png',
+    'x-pack/test/functional/failure_debug/html/*.html',
+    'x-pack/test/functional/apps/reporting/reports/session/*.pdf',
+  ]
+
+  ARTIFACT_PATTERNS.each { pattern ->
+    uploadGcsArtifact(workerName, pattern)
+  }
+}
+
+def publishJunit() {
+  junit(testResults: 'target/junit/**/*.xml', allowEmptyResults: true, keepLongStdio: true)
+}
+
+def sendMail() {
+  sendInfraMail()
+  sendKibanaMail()
+}
+
+def sendInfraMail() {
+  catchError {
+    step([
+      $class: 'Mailer',
+      notifyEveryUnstableBuild: true,
+      recipients: '[email protected]',
+      sendToIndividuals: false
+    ])
+  }
+}
+
+def sendKibanaMail() {
+  catchError {
+    if(params.NOTIFY_ON_FAILURE && currentBuild.result != 'SUCCESS' && currentBuild.result != 'ABORTED') {
+      emailext(
+        // to: '[email protected]',
+        to: '[email protected]', // TODO switch this out after testing
+        subject: "${env.PROJECT_NAME} - Build # ${env.BUILD_NUMBER} - ${currentBuild.result}",
+        body: '${SCRIPT,template="groovy-html.template"}',
+        mimeType: 'text/html',
+      )
+    }
+  }
+}
+
+def runbld(script) {
+  sh '#!/usr/local/bin/runbld\n' + script
+}
+
+def bash(script) {
+  sh "#!/bin/bash -x\n${script}"
+}
+
+def doSetup() {
+  runbld "./test/scripts/jenkins_setup.sh"
+}
+
+def buildOss() {
+  runbld "./test/scripts/jenkins_build_kibana.sh"
+}
+
+def buildXpack() {
+  runbld "./test/scripts/jenkins_xpack_build_kibana.sh"
+}

packages/kbn-test/src/es/es_test_cluster.js

@@ -54,6 +54,7 @@ export function createEsTestCluster(options = {}) {
     basePath,
     esArgs,
   };
+  const transportPort = esTestConfig.getTransportPort();
 
   const cluster = new Cluster({ log, ssl });
 
@@ -88,6 +89,7 @@ export function createEsTestCluster(options = {}) {
           `cluster.name=${clusterName}`,
           `http.port=${port}`,
           'discovery.type=single-node',
+          `transport.port=${transportPort}`,
           ...esArgs,
         ],
         esEnvVars,

packages/kbn-test/src/es/es_test_config.js

@@ -38,6 +38,10 @@ export const esTestConfig = new (class EsTestConfig {
     return process.env.TEST_ES_FROM || 'snapshot';
   }
 
+  getTransportPort() {
+    return process.env.TEST_ES_TRANSPORT_PORT || '9300-9400';
+  }
+
   getUrlParts() {
     // Allow setting one complete TEST_ES_URL for Es like https://elastic:changeme@myCloudInstance:9200
     if (process.env.TEST_ES_URL) {

packages/kbn-test/src/mocha/__tests__/junit_report_generation.js

@@ -50,7 +50,10 @@ describe('dev/mocha/junit report generation', () => {
     mocha.addFile(resolve(PROJECT_DIR, 'test.js'));
     await new Promise(resolve => mocha.run(resolve));
     const report = await fcb(cb =>
-      parseString(readFileSync(resolve(PROJECT_DIR, 'target/junit/TEST-test.xml')), cb)
+      parseString(
+        readFileSync(resolve(PROJECT_DIR, 'target/junit', process.env.JOB || '.', 'TEST-test.xml')),
+        cb
+      )
     );
 
     // test case results are wrapped in <testsuites></testsuites>

packages/kbn-test/src/mocha/junit_report_generation.js

@@ -136,7 +136,13 @@ export function setupJUnitReportGeneration(runner, options = {}) {
       }
     });
 
-    const reportPath = resolve(rootDirectory, `target/junit/TEST-${reportName}.xml`);
+    const reportPath = resolve(
+      rootDirectory,
+      'target/junit',
+      process.env.JOB || '.',
+      `TEST-${reportName}.xml`
+    );
+
     const reportXML = builder.end({
       pretty: true,
       indent: '  ',