Merge branch 'main' into 2023_09_18-remove_d3_cloud_dep

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -0,0 +1,58 @@
+# https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/
+agents:
+  queue: kibana-default
+
+steps:
+  - label: "Annotate runtime parameters"
+    command: |
+      buildkite-agent annotate --context es-serverless-image --style info "ES Serverless image: $ES_SERVERLESS_IMAGE"
+      buildkite-agent annotate --context kibana-commit --style info "Kibana build hash: $BUILDKITE_BRANCH / $BUILDKITE_COMMIT"
+
+  - group: "(:kibana: x :elastic:) Trigger Kibana Serverless suite"
+    if: "build.env('SKIP_VERIFICATION') != '1' && build.env('SKIP_VERIFICATION') != 'true'"
+    steps:
+      - label: "Pre-Build"
+        command: .buildkite/scripts/lifecycle/pre_build.sh
+        key: pre-build
+        timeout_in_minutes: 10
+        agents:
+          queue: kibana-default
+
+      - label: "Build Kibana Distribution and Plugins"
+        command: .buildkite/scripts/steps/build_kibana.sh
+        agents:
+          queue: n2-16-spot
+        key: build
+        depends_on: pre-build
+        if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+        timeout_in_minutes: 60
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - label: "Pick Test Group Run Order"
+        command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+        agents:
+          queue: kibana-default
+        env:
+          FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh'
+          FTR_CONFIG_PATTERNS: '**/test_serverless/**'
+          LIMIT_CONFIG_TYPE: 'functional'
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+  - wait: ~
+
+  - label: ":arrow_up::elastic::arrow_up: Promote docker image"
+    command: .buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh $ES_SERVERLESS_IMAGE
+
+  - wait: ~
+
+  - label: 'Post-Build'
+    command: .buildkite/scripts/lifecycle/post_build.sh
+    timeout_in_minutes: 10
+    agents:
+      queue: kibana-default

.buildkite/pipelines/on_merge.yml

@@ -154,8 +154,11 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/check_types.sh
-    label: 'Check Types'
+  - command: .buildkite/scripts/steps/check_types_commits.sh
+    label: 'Check Types Commit Diff'
+    # TODO: Enable in #166813 after fixing types
+  # - command: .buildkite/scripts/steps/check_types.sh
+    # label: 'Check Types'
     agents:
       queue: n2-16-spot
     timeout_in_minutes: 60

.buildkite/pipelines/pull_request/base.yml

@@ -136,12 +136,13 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/check_types.sh
-    label: 'Check Types'
-    agents:
-      queue: n2-16-spot
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+# TODO: Enable in #166813 after fixing types
+#  - command: .buildkite/scripts/steps/check_types.sh
+#    label: 'Check Types'
+#    agents:
+#      queue: n2-16-spot
+#    timeout_in_minutes: 60
+#    retry:
+#      automatic:
+#        - exit_status: '-1'
+#          limit: 3

.buildkite/pipelines/pull_request/type_check.yml

@@ -0,0 +1,10 @@
+steps:
+  - command: .buildkite/scripts/steps/check_types.sh
+    label: 'Check Types'
+    agents:
+      queue: n2-16-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/pull_request/type_check_selective.yml

@@ -0,0 +1,10 @@
+steps:
+  - command: .buildkite/scripts/steps/check_types_commits.sh
+    label: 'Check Types Commit Diff'
+    agents:
+      queue: n2-16-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -59,6 +59,12 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/kbn_handlebars.yml'));
     }
 
+    if (GITHUB_PR_LABELS.includes('ci:hard-typecheck')) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/type_check.yml'));
+    } else {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/type_check_selective.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([
         /^src\/plugins\/controls/,

.buildkite/scripts/steps/build_api_docs.sh

@@ -4,8 +4,9 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-echo "--- Run scripts/type_check to ensure that all build available"
-node scripts/type_check
+# TODO: Enable in #166813 after fixing types
+# echo "--- Run scripts/type_check to ensure that all build available"
+# node scripts/type_check
 
 echo "--- Build API Docs"
 node --max-old-space-size=12000 scripts/build_api_docs

.buildkite/scripts/steps/check_types_commits.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+
+if [[ "${CI-}" == "true" ]]; then
+  .buildkite/scripts/bootstrap.sh
+
+  sha1="${GITHUB_PR_TARGET_BRANCH-}"
+  sha2="${GITHUB_PR_TRIGGERED_SHA-}"
+else
+  # Script take between 0 and 2 arguments representing two commit SHA's:
+  # If 0, it will diff HEAD and HEAD^
+  # If 1 (SHA1), it will diff SHA1 and SHA1^
+  # If 2 (SHA1, SHA2), it will diff SHA1 and SHA2
+  sha1="${1-HEAD}"
+  sha2="${2-$sha1^}"
+fi
+
+uniq_dirs=()
+uniq_tsconfigs=()
+
+echo "Detecting files changed between $sha1 and $sha2..."
+
+files=($(git diff --name-only $sha1 $sha2))
+
+add_dir () {
+  new_dir=$1
+
+  if [ ${#uniq_dirs[@]} -gt 0 ]; then
+    for dir in "${uniq_dirs[@]}"
+    do
+      if [[ "$new_dir" == "$dir" ]]; then
+        return
+      fi
+    done
+  fi
+
+  uniq_dirs+=($new_dir)
+}
+
+add_tsconfig () {
+  new_tsconfig=$1
+
+  if [ ${#uniq_tsconfigs[@]} -gt 0 ]; then
+    for tsconfig in "${uniq_tsconfigs[@]}"
+    do
+      if [[ "$new_tsconfig" == "$tsconfig" ]]; then
+        return
+      fi
+    done
+  fi
+
+  echo "  $new_tsconfig"
+  uniq_tsconfigs+=($new_tsconfig)
+}
+
+contains_tsconfig () {
+  dir=$1
+  tsconfig="$dir/tsconfig.json"
+  if [ -f "$tsconfig" ]; then
+    true
+  else
+    false
+  fi
+}
+
+find_tsconfig () {
+  dir=$1
+
+  if [[ "$dir" == "." ]]; then
+    return
+  fi
+
+  if contains_tsconfig $dir; then
+    add_tsconfig "$dir/tsconfig.json"
+  else
+    find_tsconfig $(dirname -- "$dir")
+  fi
+}
+
+if [ ${#files[@]} -eq 0 ]; then
+  echo "No files found!"
+  exit
+fi
+
+for file in "${files[@]}"
+do
+  dir=$(dirname -- "$file")
+
+  # Ignore buildkite dir because it traverses many kbn packages and emits incorrect results
+  if [[ "$dir" != .buildkite* ]]; then
+    add_dir $dir
+  fi
+done
+
+echo "Looking for related tsconfig.json files..."
+
+for dir in "${uniq_dirs[@]}"
+do
+  find_tsconfig $dir
+done
+
+if [ ${#uniq_tsconfigs[@]} -eq 0 ]; then
+  echo "No tsconfig.json files found for changes in $sha1 $sha2"
+  exit
+fi
+
+echo "Running scripts/type_check for each found tsconfig.json file..."
+
+for tsconfig in "${uniq_tsconfigs[@]}"
+do
+  node scripts/type_check --project $tsconfig
+done

.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+BASE_ES_SERVERLESS_REPO=docker.elastic.co/elasticsearch-ci/elasticsearch-serverless
+TARGET_IMAGE=docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
+
+ES_SERVERLESS_BUCKET=kibana-ci-es-serverless-images
+MANIFEST_FILE_NAME=latest-verified.json
+
+SOURCE_IMAGE_OR_TAG=$1
+if [[ $SOURCE_IMAGE_OR_TAG =~ :[a-zA-Z_-]+$ ]]; then
+  # $SOURCE_IMAGE_OR_TAG was a full image
+  SOURCE_IMAGE=$SOURCE_IMAGE_OR_TAG
+else
+  # $SOURCE_IMAGE_OR_TAG was an image tag
+  SOURCE_IMAGE="$BASE_ES_SERVERLESS_REPO:$SOURCE_IMAGE_OR_TAG"
+fi
+
+echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
+
+echo "Re-tagging $SOURCE_IMAGE -> $TARGET_IMAGE"
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+docker pull "$SOURCE_IMAGE"
+docker tag "$SOURCE_IMAGE" "$TARGET_IMAGE"
+docker push "$TARGET_IMAGE"
+
+ORIG_IMG_DATA=$(docker inspect "$SOURCE_IMAGE")
+ELASTIC_COMMIT_HASH=$(echo $ORIG_IMG_DATA | jq -r '.[].Config.Labels["org.opencontainers.image.revision"]')
+
+docker logout docker.elastic.co
+
+echo "Image push to $TARGET_IMAGE successful."
+echo "Promotion successful! Henceforth, thou shall be named Sir $TARGET_IMAGE"
+
+MANIFEST_UPLOAD_PATH="Skipped"
+if [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+  echo "--- Uploading latest-verified manifest to GCS"
+  cat << EOT >> $MANIFEST_FILE_NAME
+{
+  "build_url": "$BUILDKITE_BUILD_URL",
+  "kibana_commit": "$BUILDKITE_COMMIT",
+  "kibana_branch": "$BUILDKITE_BRANCH",
+  "elasticsearch_serverless_tag": "$SOURCE_IMAGE_OR_TAG",
+  "elasticsearch_serverless_image_url: "$SOURCE_IMAGE",
+  "elasticsearch_serverless_commit": "TODO: this currently can't be decided",
+  "elasticsearch_commit": "$ELASTIC_COMMIT_HASH",
+  "created_at": "`date`",
+  "timestamp": "`FORCE_COLOR=0 node -p 'Date.now()'`"
+}
+EOT
+
+  gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" \
+    cp $MANIFEST_FILE_NAME "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
+  gsutil acl ch -u AllUsers:R "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
+  MANIFEST_UPLOAD_PATH="<a href=\"https://storage.googleapis.com/$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME\">$MANIFEST_FILE_NAME</a>"
+
+elif [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ ]]; then
+  echo "--- Skipping upload of latest-verified manifest to GCS, ES Serverless build tag is not pointing to a hash"
+elif [[ "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+  echo "--- Skipping upload of latest-verified manifest to GCS, flag was not provided"
+fi
+
+echo "--- Annotating build with info"
+cat << EOT | buildkite-agent annotate --style "success"
+  <h2>Promotion successful!</h2>
+  <br/>New image: $TARGET_IMAGE
+  <br/>Source image: $SOURCE_IMAGE
+  <br/>Kibana commit: <a href="https://github.com/elastic/kibana/commit/$BUILDKITE_COMMIT">$BUILDKITE_COMMIT</a>
+  <br/>Elasticsearch commit: <a href="https://github.com/elastic/elasticsearch/commit/$ELASTIC_COMMIT_HASH">$ELASTIC_COMMIT_HASH</a>
+  <br/>Manifest file: $MANIFEST_UPLOAD_PATH
+EOT

api_docs/actions.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/actions
 title: "actions"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the actions plugin
-date: 2023-09-25
+date: 2023-09-26
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
 ---
 import actionsObj from './actions.devdocs.json';

api_docs/advanced_settings.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/advancedSettings
 title: "advancedSettings"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the advancedSettings plugin
-date: 2023-09-25
+date: 2023-09-26
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
 ---
 import advancedSettingsObj from './advanced_settings.devdocs.json';

api_docs/aiops.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiops
 title: "aiops"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiops plugin
-date: 2023-09-25
+date: 2023-09-26
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops']
 ---
 import aiopsObj from './aiops.devdocs.json';

api_docs/alerting.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/alerting
 title: "alerting"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the alerting plugin
-date: 2023-09-25
+date: 2023-09-26
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'alerting']
 ---
 import alertingObj from './alerting.devdocs.json';

api_docs/apm.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apm
 title: "apm"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apm plugin
-date: 2023-09-25
+date: 2023-09-26
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apm']
 ---
 import apmObj from './apm.devdocs.json';

api_docs/apm_data_access.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apmDataAccess
 title: "apmDataAccess"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apmDataAccess plugin
-date: 2023-09-25
+date: 2023-09-26
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apmDataAccess']
 ---
 import apmDataAccessObj from './apm_data_access.devdocs.json';

api_docs/asset_manager.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/assetManager
 title: "assetManager"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the assetManager plugin
-date: 2023-09-25
+date: 2023-09-26
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'assetManager']
 ---
 import assetManagerObj from './asset_manager.devdocs.json';