Merge branch 'master' into mv/kibana-keystore

.backportrc.json

@@ -25,6 +25,7 @@
   ],
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
+    "^v8.0.0$": "master",
     "^v7.9.0$": "7.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   }

.ci/Jenkinsfile_coverage

@@ -23,18 +23,25 @@ kibanaPipeline(timeoutMinutes: 240) {
 }
 
 def handleIngestion(timestamp) {
+  def previousSha = handlePreviousSha()
   kibanaPipeline.downloadCoverageArtifacts()
   kibanaCoverage.prokLinks("### Process HTML Links")
   kibanaCoverage.collectVcsInfo("### Collect VCS Info")
   kibanaCoverage.generateReports("### Merge coverage reports")
   kibanaCoverage.uploadCombinedReports()
-  kibanaCoverage.ingest(timestamp, '### Injest && Upload')
+  kibanaCoverage.ingest(env.JOB_NAME, BUILD_NUMBER, BUILD_URL, timestamp, previousSha, '### Ingest && Upload')
   kibanaCoverage.uploadCoverageStaticSite(timestamp)
 }
 
+def handlePreviousSha() {
+  def previous = kibanaCoverage.downloadPrevious('### Download OLD Previous')
+  kibanaCoverage.uploadPrevious('### Upload NEW Previous')
+  return previous
+}
+
 def handleFail() {
   def buildStatus = buildUtils.getBuildStatus()
-  if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED') {
+  if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED' && buildStatus != 'UNSTABLE') {
     slackNotifications.sendFailedBuild(
       channel: '#kibana-qa',
       username: 'Kibana QA'

.ci/end2end.groovy

@@ -14,6 +14,7 @@ pipeline {
     HOME = "${env.WORKSPACE}"
     E2E_DIR = 'x-pack/plugins/apm/e2e'
     PIPELINE_LOG_LEVEL = 'DEBUG'
+    KBN_OPTIMIZER_THEMES = 'v7light'
   }
   options {
     timeout(time: 1, unit: 'HOURS')
@@ -76,7 +77,7 @@ pipeline {
         }
       }
       steps{
-        notifyStatus('Running smoke tests', 'PENDING')
+        notifyTestStatus('Running smoke tests', 'PENDING')
         dir("${BASE_DIR}"){
           sh "${E2E_DIR}/ci/run-e2e.sh"
         }
@@ -95,10 +96,10 @@ pipeline {
           }
         }
         unsuccessful {
-          notifyStatus('Test failures', 'FAILURE')
+          notifyTestStatus('Test failures', 'FAILURE')
         }
         success {
-          notifyStatus('Tests passed', 'SUCCESS')
+          notifyTestStatus('Tests passed', 'SUCCESS')
         }
       }
     }
@@ -113,5 +114,9 @@ pipeline {
 }
 
 def notifyStatus(String description, String status) {
-  withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanDisplayURL())
+  withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanTabURL('pipeline'))
+}
+
+def notifyTestStatus(String description, String status) {
+  withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanTabURL('tests'))
 }

.ci/es-snapshots/Jenkinsfile_build_es

@@ -25,7 +25,7 @@ def PROMOTE_WITHOUT_VERIFY = !!params.PROMOTE_WITHOUT_VERIFICATION
 timeout(time: 120, unit: 'MINUTES') {
   timestamps {
     ansiColor('xterm') {
-      node(workers.label('s')) {
+      node(workers.label('l')) {
         catchErrors {
           def VERSION
           def SNAPSHOT_ID
@@ -154,9 +154,10 @@ def buildArchives(destination) {
     "NODE_NAME=",
   ]) {
     sh """
-      ./gradlew -p distribution/archives assemble --parallel
+      ./gradlew -Dbuild.docker=true assemble --parallel
       mkdir -p ${destination}
-      find distribution/archives -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -exec cp {} ${destination} \\;
+      find distribution -type f \\( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \\) -not -path *no-jdk* -not -path *build-context* -exec cp {} ${destination} \\;
+      docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:\${0} | gzip > ${destination}/elasticsearch-\${0}-docker-image.tar.gz'
     """
   }
 }

.ci/packer_cache.sh

@@ -2,59 +2,5 @@
 
 set -e
 
-branch="$(git rev-parse --abbrev-ref HEAD 2> /dev/null)"
-
-# run setup script that gives us node, yarn, and bootstraps the project
-source src/dev/ci_setup/setup.sh;
-
-# download es snapshots
-node scripts/es snapshot --download-only;
-node scripts/es snapshot --license=oss --download-only;
-
-# download reporting browsers
-(cd "x-pack" && yarn gulp prepare);
-
-# cache the chromedriver archive
-chromedriverDistVersion="$(node -e "console.log(require('chromedriver').version)")"
-chromedriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.chromedriver)")"
-if [ -z "$chromedriverDistVersion" ] || [ -z "$chromedriverPkgVersion" ]; then
-  echo "UNABLE TO DETERMINE CHROMEDRIVER VERSIONS"
-  exit 1
-fi
-mkdir -p .chromedriver
-curl "https://chromedriver.storage.googleapis.com/$chromedriverDistVersion/chromedriver_linux64.zip" > .chromedriver/chromedriver.zip
-echo "$chromedriverPkgVersion" > .chromedriver/pkgVersion
-
-# cache the geckodriver archive
-geckodriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.geckodriver)")"
-if [ -z "$geckodriverPkgVersion" ]; then
-  echo "UNABLE TO DETERMINE geckodriver VERSIONS"
-  exit 1
-fi
-mkdir -p ".geckodriver"
-cp "node_modules/geckodriver/geckodriver.tar.gz" .geckodriver/geckodriver.tar.gz
-echo "$geckodriverPkgVersion" > .geckodriver/pkgVersion
-
-echo "Creating bootstrap_cache archive"
-
-# archive cacheable directories
-mkdir -p "$HOME/.kibana/bootstrap_cache"
-tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \
-  x-pack/plugins/reporting/.chromium \
-  .es \
-  .chromedriver \
-  .geckodriver;
-
-echo "Adding node_modules"
-# Find all of the node_modules directories that aren't test fixtures, and aren't inside other node_modules directories, and append them to the tar
-find . -type d -name node_modules -not -path '*__fixtures__*' -prune -print0 | xargs -0I % tar -rf "$HOME/.kibana/bootstrap_cache/$branch.tar" "%"
-
-echo "created $HOME/.kibana/bootstrap_cache/$branch.tar"
-
-if [ "$branch" == "master" ]; then
-  echo "Creating bootstrap cache for 7.x";
-
-  git clone https://github.com/elastic/kibana.git --branch 7.x --depth 1 /tmp/kibana-7.x
-  (cd /tmp/kibana-7.x && ./.ci/packer_cache.sh);
-  rm -rf /tmp/kibana-7.x;
-fi
+./.ci/packer_cache_for_branch.sh master
+./.ci/packer_cache_for_branch.sh 7.x

.ci/packer_cache_for_branch.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -e
+
+branch="$1"
+checkoutDir="$(pwd)"
+
+if [[ "$branch" != "master" ]]; then
+  checkoutDir="/tmp/kibana-$branch"
+  git clone https://github.com/elastic/kibana.git --branch "$branch" --depth 1 "$checkoutDir"
+  cd "$checkoutDir"
+fi
+
+source src/dev/ci_setup/setup.sh;
+
+# download es snapshots
+node scripts/es snapshot --download-only;
+node scripts/es snapshot --license=oss --download-only;
+
+# download reporting browsers
+(cd "x-pack" && yarn gulp prepare);
+
+# cache the chromedriver archive
+chromedriverDistVersion="$(node -e "console.log(require('chromedriver').version)")"
+chromedriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.chromedriver)")"
+if [ -z "$chromedriverDistVersion" ] || [ -z "$chromedriverPkgVersion" ]; then
+  echo "UNABLE TO DETERMINE CHROMEDRIVER VERSIONS"
+  exit 1
+fi
+mkdir -p .chromedriver
+curl "https://chromedriver.storage.googleapis.com/$chromedriverDistVersion/chromedriver_linux64.zip" > .chromedriver/chromedriver.zip
+echo "$chromedriverPkgVersion" > .chromedriver/pkgVersion
+
+# cache the geckodriver archive
+geckodriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.geckodriver)")"
+if [ -z "$geckodriverPkgVersion" ]; then
+  echo "UNABLE TO DETERMINE geckodriver VERSIONS"
+  exit 1
+fi
+mkdir -p ".geckodriver"
+cp "node_modules/geckodriver/geckodriver.tar.gz" .geckodriver/geckodriver.tar.gz
+echo "$geckodriverPkgVersion" > .geckodriver/pkgVersion
+
+echo "Creating bootstrap_cache archive"
+
+# archive cacheable directories
+mkdir -p "$HOME/.kibana/bootstrap_cache"
+tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \
+  x-pack/plugins/reporting/.chromium \
+  .es \
+  .chromedriver \
+  .geckodriver;
+
+echo "Adding node_modules"
+# Find all of the node_modules directories that aren't test fixtures, and aren't inside other node_modules directories, and append them to the tar
+find . -type d -name node_modules -not -path '*__fixtures__*' -prune -print0 | xargs -0I % tar -rf "$HOME/.kibana/bootstrap_cache/$branch.tar" "%"
+
+echo "created $HOME/.kibana/bootstrap_cache/$branch.tar"
+
+if [[ "$branch" != "master" ]]; then
+  rm --preserve-root -rf "$checkoutDir"
+fi

.ci/pipeline-library/README.md

@@ -0,0 +1,8 @@
+# Kibana Jenkins Pipeline Library
+
+## Running tests
+
+```bash
+cd .ci/pipeline-library
+./gradlew test
+```

.ci/pipeline-library/build.gradle

@@ -0,0 +1,46 @@
+plugins {
+  id 'groovy'
+  id 'idea'
+}
+
+group = 'co.elastic.kibana.pipeline'
+version = '0.0.1'
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+  maven { url 'https://repo.jenkins-ci.org/releases/' }
+  maven { url 'https://repo.maven.apache.org/maven2' }
+}
+
+dependencies {
+  implementation 'org.codehaus.groovy:groovy-all:2.4.12'
+  implementation 'org.jenkins-ci.main:jenkins-core:2.23'
+  implementation 'org.jenkins-ci.plugins.workflow:workflow-step-api:2.19@jar'
+  testImplementation 'com.lesfurets:jenkins-pipeline-unit:1.4'
+  testImplementation 'junit:junit:4.12'
+  testImplementation 'org.mockito:mockito-core:2.+'
+  testImplementation 'org.assertj:assertj-core:3.15+' // Temporary https://github.com/jenkinsci/JenkinsPipelineUnit/issues/209
+}
+
+sourceSets {
+  main {
+    groovy {
+      srcDirs = ['vars']
+    }
+  }
+
+  test {
+    groovy {
+      srcDirs = ['src/test']
+    }
+  }
+}
+
+test {
+  testLogging {
+    events 'passed', 'skipped', 'failed'
+    exceptionFormat = 'full'
+  }
+}

.ci/pipeline-library/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists