Merge remote-tracking branch 'upstream/master' into visualizations_93770

.backportrc.json

@@ -3,6 +3,8 @@
   "targetBranchChoices": [
     { "name": "master", "checked": true },
     { "name": "7.x", "checked": true },
+    "7.15",
+    "7.14",
     "7.13",
     "7.12",
     "7.11",
@@ -31,7 +33,7 @@
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
     "^v8.0.0$": "master",
-    "^v7.14.0$": "7.x",
+    "^v7.16.0$": "7.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.bazelrc.common

@@ -94,6 +94,12 @@ build --nolegacy_external_runfiles
 run --nolegacy_external_runfiles
 test --nolegacy_external_runfiles
 
+# Runfiles should be enabled un order to use `exports_directories_only` in the yarn_install rule
+# https://bazelbuild.github.io/rules_nodejs/Built-ins.html#yarn_install-exports_directories_only
+build --enable_runfiles
+run --enable_runfiles
+test --enable_runfiles
+
 # Turn on --incompatible_strict_action_env which was on by default
 # in Bazel 0.21.0 but turned off again in 0.22.0. Follow
 # https://github.com/bazelbuild/bazel/issues/7026 for more details.
@@ -104,10 +110,11 @@ build --incompatible_strict_action_env
 run --incompatible_strict_action_env
 test --incompatible_strict_action_env
 
+# DISABLED for now due to https://bazelbuild.github.io/rules_nodejs/Built-ins.html#yarn_install-exports_directories_only
 # Do not build runfile trees by default. If an execution strategy relies on runfile
 # symlink tree, the tree is created on-demand. See: https://github.com/bazelbuild/bazel/issues/6627
 # and https://github.com/bazelbuild/bazel/commit/03246077f948f2790a83520e7dccc2625650e6df
-build --nobuild_runfile_links
+# build --nobuild_runfile_links
 
 # When running `bazel coverage` --instrument_test_targets needs to be set in order to
 # collect coverage information from test targets

.buildkite/README.md

@@ -0,0 +1,10 @@
+# Kibana / Buildkite
+
+## Directory Structure
+
+- `hooks` - special directory used by Buildkite agents for [hooks](https://buildkite.com/docs/agent/v3/hooks)
+- `pipelines` - contains pipeline definitions
+- `scripts/common` - scripts that get `source`d by other scripts to set environment variables or import shared functions
+- `scripts/lifecycle` - general scripts for tasks that run before or after individual steps or the entire build
+- `scripts/steps` - scripts that define something that will run for a step defined in a pipeline
+- `scripts/*` - all other scripts are building blocks that make up the tasks in pipelines. They may be run by other scripts, but should not be `source`d

.buildkite/agents.json

@@ -0,0 +1,79 @@
+{
+  "gcp": {
+    "project": "elastic-kibana-ci",
+    "zones": ["us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"],
+    "serviceAccount": "elastic-buildkite-agent@elastic-kibana-ci.iam.gserviceaccount.com",
+    "imageFamily": "kb-ubuntu",
+    "subnetwork": "buildkite",
+    "disableExternalIp": true,
+    "diskType": "pd-ssd",
+    "diskSizeGb": 75,
+    "overprovision": 0,
+    "minimumAgents": 0,
+    "maximumAgents": 50,
+    "gracefulStopAfterMins": 360,
+    "hardStopAfterMins": 540,
+    "idleTimeoutMins": 10,
+    "exitAfterOneJob": false,
+
+    "agents": [
+      {
+        "queue": "default",
+        "name": "kb-default",
+        "minimumAgents": 1,
+        "maximumAgents": 100,
+        "idleTimeoutMins": 60,
+        "machineType": "e2-small"
+      },
+      {
+        "queue": "c2-8",
+        "name": "kb-c2-8",
+        "machineType": "c2-standard-8",
+        "localSsds": 1
+      },
+      {
+        "queue": "c2-4",
+        "name": "kb-c2-4",
+        "machineType": "c2-standard-4",
+        "localSsds": 1
+      },
+      {
+        "queue": "jest",
+        "name": "kb-jest",
+        "machineType": "n2-standard-2",
+        "diskSizeGb": 128
+      },
+      {
+        "queue": "ci-group",
+        "name": "kb-cigroup",
+        "machineType": "n2-standard-8",
+        "diskSizeGb": 256
+      },
+      {
+        "queue": "ci-group-4",
+        "name": "kb-cigroup-4",
+        "machineType": "n2-standard-4",
+        "diskSizeGb": 128
+      },
+      {
+        "queue": "ci-group-4d",
+        "name": "kb-cigroup-4d",
+        "machineType": "n2d-standard-4",
+        "diskSizeGb": 128
+      },
+      {
+        "queue": "ci-group-6",
+        "name": "kb-cigroup-6",
+        "machineType": "n2-custom-6-16384",
+        "diskSizeGb": 128
+      },
+      {
+        "queue": "packer",
+        "name": "kb-packer",
+        "serviceAccount": "[email protected]",
+        "maximumAgents": 10,
+        "machineType": "e2-small"
+      }
+    ]
+  }
+}

.buildkite/hooks/pre-command

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/lifecycle/pre_command.sh

.buildkite/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "kibana-buildkite",
+  "version": "1.0.0",
+  "private": true,
+  "dependencies": {
+    "kibana-buildkite-library": "elastic/kibana-buildkite-library"
+  }
+}

.buildkite/pipelines/on_merge.yml

@@ -0,0 +1,24 @@
+env:
+  GITHUB_COMMIT_STATUS_ENABLED: 'true'
+  GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/on-merge'
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
+    label: Default Build and Metrics
+    agents:
+      queue: c2-8
+
+  - command: .buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh
+    label: Build TS Refs and Check Public API Docs
+    agents:
+      queue: c2-4
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build

.buildkite/pipelines/pull_request.yml

@@ -0,0 +1,17 @@
+env:
+  GITHUB_COMMIT_STATUS_ENABLED: 'true'
+  GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/kibana-pull-request'
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: echo 'Hello World'
+    label: Test
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build

.buildkite/scripts/bootstrap.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo "--- yarn install and bootstrap"
+yarn kbn bootstrap --verbose
+
+###
+### upload ts-refs-cache artifacts as quickly as possible so they are available for download
+###
+if [[ "${BUILD_TS_REFS_CACHE_CAPTURE:-}" == "true" ]]; then
+  echo "--- Upload ts-refs-cache"
+  cd "$KIBANA_DIR/target/ts_refs_cache"
+  gsutil cp "*.zip" 'gs://kibana-ci-ts-refs-cache/'
+  cd "$KIBANA_DIR"
+fi
+
+if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
+  verify_no_git_changes 'yarn kbn bootstrap'
+fi

.buildkite/scripts/build_kibana.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export KBN_NP_PLUGINS_BUILT=true
+
+echo "--- Build Kibana Distribution"
+node scripts/build --debug
+
+echo "--- Archive Kibana Distribution"
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$KIBANA_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+mkdir -p "$KIBANA_BUILD_LOCATION"
+cp -pR install/kibana/. "$KIBANA_BUILD_LOCATION/"

.buildkite/scripts/build_kibana_plugins.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "--- Build Platform Plugins"
+node scripts/build_kibana_platform_plugins \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --scan-dir "$KIBANA_DIR/examples" \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
+  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
+  --scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
+  --scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
+  --scan-dir "$XPACK_DIR/examples" \
+  --verbose
+
+echo "--- Archive built plugins"
+shopt -s globstar
+tar -zcf \
+  target/kibana-default-plugins.tar.gz \
+  x-pack/plugins/**/target/public \
+  x-pack/test/**/target/public \
+  examples/**/target/public \
+  x-pack/examples/**/target/public \
+  test/**/target/public

.buildkite/scripts/common/env.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+export CI=true
+
+KIBANA_DIR=$(pwd)
+export KIBANA_DIR
+export XPACK_DIR="$KIBANA_DIR/x-pack"
+
+export CACHE_DIR="$HOME/.kibana"
+PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
+export PARENT_DIR
+export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"
+
+KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
+export KIBANA_PKG_BRANCH
+export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
+
+export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
+export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
+export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
+export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress"
+
+export NODE_OPTIONS="--max-old-space-size=4096"
+
+export FORCE_COLOR=1
+export TEST_BROWSER_HEADLESS=1
+
+export ELASTIC_APM_ENVIRONMENT=ci
+export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
+
+CI_REPORTING_ENABLED=false # TODO enable when ready, only controls checks reporter and APM
+
+if is_pr; then
+  export ELASTIC_APM_ACTIVE=false
+  export CHECKS_REPORTER_ACTIVE="${CI_REPORTING_ENABLED-}"
+
+  # These can be removed once we're not supporting Jenkins and Buildkite at the same time
+  # These are primarily used by github checks reporter and can be configured via /github_checks_api.json
+  export ghprbGhRepository="elastic/kibana"
+  export ghprbActualCommit="$BUILDKITE_COMMIT"
+  export BUILD_URL="$BUILDKITE_BUILD_URL"
+
+  # set_git_merge_base # TODO for PRs
+else
+  export ELASTIC_APM_ACTIVE="${CI_REPORTING_ENABLED-}"
+  export CHECKS_REPORTER_ACTIVE=false
+fi
+
+export FLEET_PACKAGE_REGISTRY_PORT=6104
+export TEST_CORS_SERVER_PORT=6105
+
+export DETECT_CHROMEDRIVER_VERSION=true
+export CHROMEDRIVER_FORCE_DOWNLOAD=true
+
+export GCS_UPLOAD_PREFIX=FAKE_UPLOAD_PREFIX # TODO remove the need for this
+
+export KIBANA_BUILD_LOCATION="$WORKSPACE/kibana-build-xpack"
+
+if [[ "${BUILD_TS_REFS_CACHE_ENABLE:-}" != "true" ]]; then
+  export BUILD_TS_REFS_CACHE_ENABLE=false
+fi
+
+export BUILD_TS_REFS_DISABLE=true
+export DISABLE_BOOTSTRAP_VALIDATION=true
+
+export TEST_KIBANA_HOST=localhost
+export TEST_KIBANA_PORT=6101
+export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101"
+export TEST_ES_URL="http://elastic:changeme@localhost:6102"
+export TEST_ES_TRANSPORT_PORT=6301-6309
+export TEST_CORS_SERVER_PORT=6106
+export ALERTING_PROXY_PORT=6105

.buildkite/scripts/common/setup_bazel.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+KIBANA_BUILDBUDDY_CI_API_KEY=$(vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
+export KIBANA_BUILDBUDDY_CI_API_KEY
+
+cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"
+
+###
+### append auth token to buildbuddy into "$HOME/.bazelrc";
+###
+echo "# Appended by .buildkite/scripts/setup_bazel.sh" >> "$HOME/.bazelrc"
+echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc"
+
+###
+### remove write permissions on buildbuddy remote cache for prs
+###
+if [[ "${BUILDKITE_PULL_REQUEST:-}" && "$BUILDKITE_PULL_REQUEST" != "false" ]] ; then
+  {
+    echo "# Uploads logs & artifacts without writing to cache"
+    echo "build --noremote_upload_local_results"
+  } >> "$HOME/.bazelrc"
+fi