Merge branch 'master' of github.com:elastic/kibana into controls/dndK…

…itDragDrop

.buildkite/hooks/post-command

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+.buildkite/scripts/lifecycle/post_command.sh

.buildkite/pipelines/es_snapshots/build.yml

@@ -0,0 +1,5 @@
+steps:
+  - command: .buildkite/scripts/steps/es_snapshots/build.sh
+    label: Build ES Snapshot
+    agents:
+      queue: c2-8

.buildkite/pipelines/es_snapshots/promote.yml

@@ -0,0 +1,12 @@
+steps:
+  - block: 'Promote'
+    prompt: "Enter the details for the snapshot you'd like to promote"
+    if: "build.env('ES_SNAPSHOT_MANIFEST') == null"
+    # Later, this could be a dropdown dynamically filled with recent builds
+    fields:
+      - text: 'ES_SNAPSHOT_MANIFEST'
+        key: 'ES_SNAPSHOT_MANIFEST'
+        hint: 'URL pointing to the manifest to promote'
+        required: true
+  - label: Promote Snapshot
+    command: .buildkite/scripts/steps/es_snapshots/promote.sh

.buildkite/pipelines/es_snapshots/verify.yml

@@ -0,0 +1,103 @@
+env:
+  IGNORE_SHIP_CI_STATS_ERROR: 'true'
+steps:
+  - block: 'Verify'
+    prompt: "Enter the details for the snapshot you'd like to verify"
+    if: "build.env('ES_SNAPSHOT_MANIFEST') == null"
+    # Later, this could be a dropdown dynamically filled with recent builds
+    fields:
+      - text: 'ES_SNAPSHOT_MANIFEST'
+        key: 'ES_SNAPSHOT_MANIFEST'
+        hint: 'URL pointing to the manifest to promote'
+        required: true
+
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      queue: c2-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
+    label: 'Default CI Group'
+    parallelism: 13
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 150
+    key: default-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
+    label: 'Docker CI Group'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    key: default-cigroup-docker
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
+    label: 'OSS CI Group'
+    parallelism: 11
+    agents:
+      queue: ci-group-4d
+    depends_on: build
+    timeout_in_minutes: 120
+    key: oss-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/jest_integration.sh
+    label: 'Jest Integration Tests'
+    agents:
+      queue: n2-4
+    timeout_in_minutes: 120
+    key: jest-integration
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/api_integration.sh
+    label: 'API Integration Tests'
+    agents:
+      queue: jest
+    timeout_in_minutes: 120
+    key: api-integration
+
+  - command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
+    label: Trigger promotion
+    depends_on:
+      - default-cigroup
+      - default-cigroup-docker
+      - oss-cigroup
+      - jest-integration
+      - api-integration
+
+  - wait: ~
+    continue_on_failure: true
+
+  - plugins:
+      - junit-annotate#v1.9.0:
+          artifacts: target/junit/**/*.xml
+          job-uuid-file-pattern: '-bk__(.*).xml'
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build

.buildkite/pipelines/hourly.yml

@@ -0,0 +1,169 @@
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      queue: c2-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
+    label: 'Default CI Group'
+    parallelism: 13
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 150
+    key: default-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
+    label: 'Docker CI Group'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    key: default-cigroup-docker
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
+    label: 'OSS CI Group'
+    parallelism: 11
+    agents:
+      queue: ci-group-4d
+    depends_on: build
+    timeout_in_minutes: 120
+    key: oss-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_accessibility.sh
+    label: 'OSS Accessibility Tests'
+    agents:
+      queue: ci-group-4d
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
+    label: 'Default Accessibility Tests'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_firefox.sh
+    label: 'OSS Firefox Tests'
+    agents:
+      queue: ci-group-4d
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
+    label: 'Default Firefox Tests'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_misc.sh
+    label: 'OSS Misc Functional Tests'
+    agents:
+      queue: ci-group-4d
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
+    label: 'Saved Object Field Metrics'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/jest_integration.sh
+    label: 'Jest Integration Tests'
+    agents:
+      queue: n2-4
+    timeout_in_minutes: 120
+    key: jest-integration
+
+  - command: .buildkite/scripts/steps/test/api_integration.sh
+    label: 'API Integration Tests'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 120
+    key: api-integration
+
+  - command: .buildkite/scripts/steps/test/jest.sh
+    label: 'Jest Tests'
+    agents:
+      queue: c2-16
+    timeout_in_minutes: 120
+    key: jest
+
+  - command: .buildkite/scripts/steps/lint.sh
+    label: 'Linting'
+    agents:
+      queue: n2-2
+    key: linting
+
+  - command: .buildkite/scripts/steps/checks.sh
+    label: 'Checks'
+    agents:
+      queue: c2-4
+    key: checks
+
+  - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
+    label: 'Build Storybooks'
+    agents:
+      queue: c2-4
+    key: storybooks
+
+  - wait: ~
+    continue_on_failure: true
+
+  - plugins:
+      - junit-annotate#v1.9.0:
+          artifacts: target/junit/**/*.xml
+          job-uuid-file-pattern: '-bk__(.*).xml'
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build

.buildkite/scripts/bootstrap.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo "--- yarn install and bootstrap"
-yarn kbn bootstrap
+retry 2 15 yarn kbn bootstrap
 
 ###
 ### upload ts-refs-cache artifacts as quickly as possible so they are available for download

.buildkite/scripts/build_kibana_plugins.sh

@@ -19,13 +19,3 @@ node scripts/build_kibana_platform_plugins \
   --scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
   --scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
   --scan-dir "$XPACK_DIR/examples"
-
-echo "--- Archive built plugins"
-shopt -s globstar
-tar -zcf \
-  target/kibana-default-plugins.tar.gz \
-  x-pack/plugins/**/target/public \
-  x-pack/test/**/target/public \
-  examples/**/target/public \
-  x-pack/examples/**/target/public \
-  test/**/target/public

.buildkite/scripts/common/env.sh

@@ -11,6 +11,13 @@ PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
 export PARENT_DIR
 export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"
 
+# A few things, such as Chrome, respect this variable
+# For many agent types, the workspace is mounted on a local ssd, so will be faster than the default tmp dir location
+if [[ -d /opt/local-ssd/buildkite ]]; then
+  export TMPDIR="/opt/local-ssd/buildkite/tmp"
+  mkdir -p "$TMPDIR"
+fi
+
 KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_BRANCH
 export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
@@ -28,11 +35,9 @@ export TEST_BROWSER_HEADLESS=1
 export ELASTIC_APM_ENVIRONMENT=ci
 export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 
-CI_REPORTING_ENABLED=false # TODO enable when ready, only controls checks reporter and APM
-
 if is_pr; then
   export ELASTIC_APM_ACTIVE=false
-  export CHECKS_REPORTER_ACTIVE="${CI_REPORTING_ENABLED-}"
+  export CHECKS_REPORTER_ACTIVE=true
 
   # These can be removed once we're not supporting Jenkins and Buildkite at the same time
   # These are primarily used by github checks reporter and can be configured via /github_checks_api.json
@@ -42,7 +47,7 @@ if is_pr; then
 
   # set_git_merge_base # TODO for PRs
 else
-  export ELASTIC_APM_ACTIVE="${CI_REPORTING_ENABLED-}"
+  export ELASTIC_APM_ACTIVE=true
   export CHECKS_REPORTER_ACTIVE=false
 fi
 

.buildkite/scripts/common/setup_bazel.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
-KIBANA_BUILDBUDDY_CI_API_KEY=$(vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
+source .buildkite/scripts/common/util.sh
+
+KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
 export KIBANA_BUILDBUDDY_CI_API_KEY
 
 cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"

.buildkite/scripts/common/util.sh

@@ -49,3 +49,28 @@ docker_run() {
 
   docker run "${args[@]}" "$@"
 }
+
+is_test_execution_step() {
+  buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'true'
+}
+
+retry() {
+  local retries=$1; shift
+  local delay=$1; shift
+  local attempts=1
+
+  until "$@"; do
+    retry_exit_status=$?
+    echo "Exited with $retry_exit_status" >&2
+    if (( retries == "0" )); then
+      return $retry_exit_status
+    elif (( attempts == retries )); then
+      echo "Failed $attempts retries" >&2
+      return $retry_exit_status
+    else
+      echo "Retrying $((retries - attempts)) more times..." >&2
+      attempts=$((attempts + 1))
+      sleep "$delay"
+    fi
+  done
+}