Merge branch 'master' into improve-es-errors-logs

.backportrc.json

@@ -3,6 +3,7 @@
   "targetBranchChoices": [
     { "name": "master", "checked": true },
     { "name": "7.x", "checked": true },
+    "7.10",
     "7.9",
     "7.8",
     "7.7",
@@ -27,7 +28,7 @@
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
     "^v8.0.0$": "master",
-    "^v7.10.0$": "7.x",
+    "^v7.11.0$": "7.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   }
 }

.browserslistrc

@@ -4,6 +4,8 @@ last 2 Chrome versions
 last 2 Safari versions
 > 0.25%
 not ie 11
+not op_mini all
+not samsung 4
 
 [dev]
 last 1 chrome versions

.ci/Dockerfile

@@ -0,0 +1,38 @@
+# NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
+# If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
+
+ARG NODE_VERSION=12.19.0
+
+FROM node:${NODE_VERSION} AS base
+
+RUN apt-get update && \
+    apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
+      libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
+      libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
+      libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
+  && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+  && apt-get update \
+  && apt-get install -y rsync jq bsdtar google-chrome-stable \
+  --no-install-recommends \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \
+  && curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \
+  && unzip vault.zip \
+  && rm vault.zip \
+  && chmod +x vault \
+  && mv vault /usr/local/bin/vault
+
+RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana
+
+COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh
+RUN chmod +x /usr/local/bin/bash_standard_lib.sh
+
+COPY ./runbld /usr/local/bin/runbld
+RUN chmod +x /usr/local/bin/runbld
+
+USER kibana

.ci/Jenkinsfile_baseline_capture

@@ -7,18 +7,22 @@ kibanaPipeline(timeoutMinutes: 120) {
   githubCommitStatus.trackBuild(params.commit, 'kibana-ci-baseline') {
     ciStats.trackBuild {
       catchError {
-        parallel([
-          'oss-visualRegression': {
-            workers.ci(name: 'oss-visualRegression', size: 's-highmem', ramDisk: true) {
-              kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1)
-            }
-          },
-          'xpack-visualRegression': {
-            workers.ci(name: 'xpack-visualRegression', size: 's-highmem', ramDisk: true) {
-              kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1)
-            }
-          },
-        ])
+        withEnv([
+          'CI_PARALLEL_PROCESS_NUMBER=1'
+        ]) {
+          parallel([
+            'oss-baseline': {
+              workers.ci(name: 'oss-baseline', size: 'l', ramDisk: true, runErrorReporter: false) {
+                kibanaPipeline.functionalTestProcess('oss-baseline', './test/scripts/jenkins_baseline.sh')()
+              }
+            },
+            'xpack-baseline': {
+              workers.ci(name: 'xpack-baseline', size: 'l', ramDisk: true, runErrorReporter: false) {
+                kibanaPipeline.functionalTestProcess('xpack-baseline', './test/scripts/jenkins_xpack_baseline.sh')()
+              }
+            },
+          ])
+        }
       }
 
       kibanaPipeline.sendMail()

.ci/Jenkinsfile_coverage

@@ -12,8 +12,12 @@ kibanaPipeline(timeoutMinutes: 240) {
     ]) {
       workers.base(name: 'coverage-worker', size: 'l', ramDisk: false, bootstrapped: false) {
         catchError {
+
+          kibanaPipeline.bash("""
+            echo '${TIME_STAMP}'
+          """, "### Print Canonical Time Stamp")
+
           kibanaCoverage.runTests()
-          kibanaTeamAssign.load('team_assignment', "### Upload Team Assignment JSON")
           handleIngestion(TIME_STAMP)
         }
         handleFail()
@@ -30,8 +34,8 @@ def handleIngestion(timestamp) {
   kibanaCoverage.collectVcsInfo("### Collect VCS Info")
   kibanaCoverage.generateReports("### Merge coverage reports")
   kibanaCoverage.uploadCombinedReports()
-  kibanaCoverage.ingest(env.JOB_NAME, BUILD_NUMBER, BUILD_URL, timestamp, previousSha, '### Ingest && Upload')
   kibanaCoverage.uploadCoverageStaticSite(timestamp)
+  kibanaCoverage.ingest(env.JOB_NAME, BUILD_NUMBER, BUILD_URL, timestamp, previousSha, teamAssignmentsPath(), '### Generate Team Assignments && Ingest')
 }
 
 def handlePreviousSha() {
@@ -42,11 +46,15 @@ def handlePreviousSha() {
 
 def handleFail() {
   def buildStatus = buildUtils.getBuildStatus()
-  if(params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED' && buildStatus != 'UNSTABLE') {
+  if (params.NOTIFY_ON_FAILURE && buildStatus != 'SUCCESS' && buildStatus != 'ABORTED' && buildStatus != 'UNSTABLE') {
     slackNotifications.sendFailedBuild(
       channel: '#kibana-qa',
       username: 'Kibana QA'
     )
   }
 }
 
+def teamAssignmentsPath() {
+  return 'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'
+}
+

.ci/Jenkinsfile_security_cypress

@@ -0,0 +1,28 @@
+#!/bin/groovy
+
+library 'kibana-pipeline-library'
+kibanaLibrary.load()
+
+kibanaPipeline(timeoutMinutes: 180) {
+  slackNotifications.onFailure(
+    disabled: !params.NOTIFY_ON_FAILURE,
+    channel: '#security-solution-slack-testing'
+  ) {
+    catchError {
+      withEnv([
+        'CI_PARALLEL_PROCESS_NUMBER=1'
+      ]) {
+        def job = 'xpack-securityCypress'
+
+        workers.ci(name: job, size: 'l', ramDisk: true) {
+          kibanaPipeline.bash('test/scripts/jenkins_xpack_build_kibana.sh', 'Build Default Distributable')
+          kibanaPipeline.functionalTestProcess(job, 'test/scripts/jenkins_security_solution_cypress.sh')()
+        }
+      }
+    }
+  }
+
+  if (params.NOTIFY_ON_FAILURE) {
+    kibanaPipeline.sendMail(to: '[email protected]')
+  }
+}

.ci/end2end.groovy

@@ -37,22 +37,31 @@ pipeline {
         deleteDir()
         gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: false,
                     shallow: false, reference: "/var/lib/jenkins/.git-references/kibana.git")
+
+        // Filter when to run based on the below reasons:
+        //  - On a PRs when:
+        //    - There are changes related to the APM UI project
+        //      - only when the owners of those changes are members of the given GitHub teams
+        //  - On merges to branches when:
+        //    - There are changes related to the APM UI project
+        //  - FORCE parameter is set to true.
         script {
+          def apm_updated = false
           dir("${BASE_DIR}"){
-            def regexps =[ "^x-pack/plugins/apm/.*" ]
-            env.APM_UPDATED = isGitRegionMatch(patterns: regexps)
+            apm_updated = isGitRegionMatch(patterns: [ "^x-pack/plugins/apm/.*" ])
+          }
+          if (isPR()) {
+            def isMember = isMemberOf(user: env.CHANGE_AUTHOR, team: ['apm-ui', 'uptime'])
+            setEnvVar('RUN_APM_E2E', params.FORCE || (apm_updated && isMember))
+          } else {
+            setEnvVar('RUN_APM_E2E', params.FORCE || apm_updated)
           }
         }
       }
     }
     stage('Prepare Kibana') {
       options { skipDefaultCheckout() }
-      when {
-        anyOf {
-          expression { return params.FORCE }
-          expression { return env.APM_UPDATED != "false" }
-        }
-      }
+      when { expression { return env.RUN_APM_E2E != "false" } }
       environment {
         JENKINS_NODE_COOKIE = 'dontKillMe'
       }
@@ -70,12 +79,7 @@ pipeline {
     }
     stage('Smoke Tests'){
       options { skipDefaultCheckout() }
-      when {
-        anyOf {
-          expression { return params.FORCE }
-          expression { return env.APM_UPDATED != "false" }
-        }
-      }
+      when { expression { return env.RUN_APM_E2E != "false" } }
       steps{
         notifyTestStatus('Running smoke tests', 'PENDING')
         dir("${BASE_DIR}"){

.ci/packer_cache_for_branch.sh

@@ -18,7 +18,7 @@ node scripts/es snapshot --download-only;
 node scripts/es snapshot --license=oss --download-only;
 
 # download reporting browsers
-(cd "x-pack" && yarn gulp downloadChromium);
+(cd "x-pack" && node ../node_modules/.bin/gulp downloadChromium);
 
 # cache the chromedriver archive
 chromedriverDistVersion="$(node -e "console.log(require('chromedriver').version)")"
@@ -51,10 +51,6 @@ tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \
   .chromedriver \
   .geckodriver;
 
-echo "Adding node_modules"
-# Find all of the node_modules directories that aren't test fixtures, and aren't inside other node_modules directories, and append them to the tar
-find . -type d -name node_modules -not -path '*__fixtures__*' -prune -print0 | xargs -0I % tar -rf "$HOME/.kibana/bootstrap_cache/$branch.tar" "%"
-
 echo "created $HOME/.kibana/bootstrap_cache/$branch.tar"
 
 if [[ "$branch" != "master" ]]; then

.ci/pipeline-library/src/test/prChanges.groovy

@@ -90,7 +90,20 @@ class PrChangesTest extends KibanaBasePipelineTest {
     props([
       githubPrs: [
         getChanges: { [
-          [filename: 'docs/developer/architecture/code-exploration.asciidoc'],
+          [filename: 'docs/developer/plugin-list.asciidoc'],
+        ] },
+      ],
+    ])
+
+    assertFalse(prChanges.areChangesSkippable())
+  }
+
+  @Test
+  void 'areChangesSkippable() with plugin readme changes'() {
+    props([
+      githubPrs: [
+        getChanges: { [
+          [filename: 'src/plugins/foo/README.asciidoc'],
         ] },
       ],
     ])

.ci/pipeline-library/src/test/slackNotifications.groovy

@@ -9,6 +9,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
     super.setUp()
 
     helper.registerAllowedMethod('slackSend', [Map.class], null)
+    prop('buildState', loadScript("vars/buildState.groovy"))
     slackNotifications = loadScript('vars/slackNotifications.groovy')
   }
 
@@ -25,13 +26,49 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
   }
 
   @Test
-  void 'sendFailedBuild() should call slackSend() with message'() {
+  void 'sendFailedBuild() should call slackSend() with an in-progress message'() {
     mockFailureBuild()
 
     slackNotifications.sendFailedBuild()
 
     def args = fnMock('slackSend').args[0]
 
+    def expected = [
+      channel: '#kibana-operations-alerts',
+      username: 'Kibana Operations',
+      iconEmoji: ':jenkins:',
+      color: 'danger',
+      message: ':hourglass_flowing_sand: elastic / kibana # master #1',
+    ]
+
+    expected.each {
+      assertEquals(it.value.toString(), args[it.key].toString())
+    }
+
+    assertEquals(
+      ":hourglass_flowing_sand: *<http://jenkins.localhost:8080/job/elastic+kibana+master/1/|elastic / kibana # master #1>*",
+      args.blocks[0].text.text.toString()
+    )
+
+    assertEquals(
+      "*Failed Steps*\n• <http://jenkins.localhost:8080|Execute test task>",
+      args.blocks[1].text.text.toString()
+    )
+
+    assertEquals(
+      "*Test Failures*\n• <https://localhost/|x-pack/test/functional/apps/fake/test·ts.Fake test &lt;Component&gt; should &amp; pass &amp;>",
+      args.blocks[2].text.text.toString()
+    )
+  }
+
+  @Test
+  void 'sendFailedBuild() should call slackSend() with message'() {
+    mockFailureBuild()
+
+    slackNotifications.sendFailedBuild(isFinal: true)
+
+    def args = fnMock('slackSend').args[0]
+
     def expected = [
       channel: '#kibana-operations-alerts',
       username: 'Kibana Operations',
@@ -65,7 +102,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
     mockFailureBuild()
     def counter = 0
     helper.registerAllowedMethod('slackSend', [Map.class], { ++counter > 1 })
-    slackNotifications.sendFailedBuild()
+    slackNotifications.sendFailedBuild(isFinal: true)
 
     def args = fnMocks('slackSend')[1].args[0]
 
@@ -88,6 +125,29 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
     )
   }
 
+  @Test
+  void 'sendFailedBuild() should call slackSend() with a channel id and timestamp on second call'() {
+    mockFailureBuild()
+    helper.registerAllowedMethod('slackSend', [Map.class], { [ channelId: 'CHANNEL_ID', ts: 'TIMESTAMP' ] })
+    slackNotifications.sendFailedBuild(isFinal: false)
+    slackNotifications.sendFailedBuild(isFinal: true)
+
+    def args = fnMocks('slackSend')[1].args[0]
+
+    def expected = [
+      channel: 'CHANNEL_ID',
+      timestamp: 'TIMESTAMP',
+      username: 'Kibana Operations',
+      iconEmoji: ':jenkins:',
+      color: 'danger',
+      message: ':broken_heart: elastic / kibana # master #1',
+    ]
+
+    expected.each {
+      assertEquals(it.value.toString(), args[it.key].toString())
+    }
+  }
+
   @Test
   void 'getTestFailures() should truncate list of failures to 10'() {
     prop('testUtils', [

.ci/runbld_no_junit.yml

@@ -3,4 +3,4 @@
 profiles:
 - ".*": # Match any job
     tests:
-      junit-filename-pattern: "8d8bd494-d909-4e67-a052-7e8b5aaeb5e4" # A bogus path that should never exist
+      junit-filename-pattern: false