Merge branch 'main' into issues/113737

# Conflicts:
#	.i18nrc.json

.backportrc.json

@@ -1,7 +1,8 @@
 {
-  "upstream": "elastic/kibana",
+  "repoOwner": "elastic",
+  "repoName": "kibana",
   "targetBranchChoices": [
-    { "name": "main", "checked": true },
+    "main",
     "8.0",
     "7.17",
     "7.16",
@@ -32,11 +33,14 @@
     "6.0",
     "5.6"
   ],
-  "targetPRLabels": ["backport"],
+  "targetPRLabels": [
+    "backport"
+  ],
   "branchLabelMapping": {
     "^v8.1.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,
-  "autoMergeMethod": "squash"
+  "autoMergeMethod": "squash",
+  "backportBinary": "node scripts/backport"
 }

.buildkite/pipelines/package_testing.yml

@@ -0,0 +1,27 @@
+steps:
+  - command: .buildkite/scripts/steps/package_testing/build.sh
+    label: Build Packages
+    agents:
+      queue: c2-16
+    timeout_in_minutes: 60
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - wait
+
+  - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
+    label: Package testing for deb
+    agents:
+      queue: n2-4-virt
+    timeout_in_minutes: 60
+
+  - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
+    label: Package testing for rpm
+    agents:
+      queue: n2-4-virt
+    timeout_in_minutes: 60
+
+  - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
+    label: Package testing for docker
+    agents:
+      queue: n2-4-virt
+    timeout_in_minutes: 60

.buildkite/pipelines/pull_request/osquery_cypress.yml

@@ -9,3 +9,5 @@ steps:
       automatic:
         - exit_status: '*'
           limit: 1
+    artifact_paths:
+      - "target/kibana-osquery/**/*"

.buildkite/scripts/common/setup_node.sh

@@ -61,7 +61,14 @@ YARN_VERSION=$(node -e "console.log(String(require('./package.json').engines.yar
 export YARN_VERSION
 
 if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then
-  npm install -g "yarn@^${YARN_VERSION}"
+  rm -rf "$(npm root -g)/yarn" # in case the directory is in a bad state
+  if [[ ! $(npm install -g "yarn@^${YARN_VERSION}") ]]; then
+    # If this command is terminated early, e.g. because the build was cancelled in buildkite,
+    # a yarn directory is left behind in a bad state that can cause all subsequent installs to fail
+    rm -rf "$(npm root -g)/yarn"
+    echo "Trying again to install yarn..."
+    npm install -g "yarn@^${YARN_VERSION}"
+  fi
 fi
 
 yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE"

.buildkite/scripts/lifecycle/pre_command.sh

@@ -9,8 +9,22 @@ export BUILDKITE_TOKEN
 
 echo '--- Install buildkite dependencies'
 cd '.buildkite'
-retry 5 15 yarn install --production --pure-lockfile
-cd -
+
+# If this yarn install is terminated early, e.g. if the build is cancelled in buildkite,
+# A node module could end up in a bad state that can cause all future builds to fail
+# So, let's cache clean and try again to make sure that's not what caused the error
+install_deps() {
+  yarn install --production --pure-lockfile
+  EXIT=$?
+  if [[ "$EXIT" != "0" ]]; then
+    yarn cache clean
+  fi
+  return $EXIT
+}
+
+retry 5 15 install_deps
+
+cd ..
 
 node .buildkite/scripts/lifecycle/print_agent_links.js || true
 

.buildkite/scripts/steps/functional/osquery_cypress.sh

@@ -4,8 +4,9 @@ set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
 
+export BUILD_TS_REFS_DISABLE=false
 .buildkite/scripts/bootstrap.sh
-.buildkite/scripts/download_build_artifacts.sh
+node scripts/build_kibana_platform_plugins.js
 
 export JOB=kibana-osquery-cypress
 
@@ -16,5 +17,5 @@ cd "$XPACK_DIR"
 checks-reporter-with-killswitch "Osquery Cypress Tests" \
  node scripts/functional_tests \
    --debug --bail \
-   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
    --config test/osquery_cypress/cli_config.ts
+

.buildkite/scripts/steps/package_testing/build.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Build Kibana Distribution"
+node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-ubi --skip-docker-contexts
+
+cd target
+buildkite-agent artifact upload "./*-docker-image.tar.gz;./*.deb;./*.rpm"
+cd ..

.buildkite/scripts/steps/package_testing/test.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Package Testing for $TEST_PACKAGE"
+
+mkdir -p target
+cd target
+if [[ "$TEST_PACKAGE" == "deb" ]]; then
+  buildkite-agent artifact download 'kibana-*.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  KIBANA_IP_ADDRESS="192.168.50.5"
+elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
+  buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  KIBANA_IP_ADDRESS="192.168.50.6"
+elif [[ "$TEST_PACKAGE" == "docker" ]]; then
+  buildkite-agent artifact download 'kibana-*-docker-image.tar.gz' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  KIBANA_IP_ADDRESS="192.168.50.7"
+fi
+cd ..
+
+export VAGRANT_CWD=test/package
+vagrant up "$TEST_PACKAGE" --no-provision
+
+node scripts/es snapshot \
+  -E network.bind_host=127.0.0.1,192.168.50.1 \
+  -E discovery.type=single-node \
+  --license=trial &
+while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
+
+vagrant provision "$TEST_PACKAGE"
+
+export TEST_BROWSER_HEADLESS=1
+export TEST_KIBANA_URL="http://elastic:changeme@$KIBANA_IP_ADDRESS:5601"
+export TEST_ES_URL=http://elastic:[email protected]:9200
+
+cd x-pack
+node scripts/functional_test_runner.js --include-tag=smoke

.buildkite/scripts/steps/storybooks/build_and_upload.js

@@ -14,12 +14,11 @@ const path = require('path');
 const STORYBOOKS = [
   'apm',
   'canvas',
-  'codeeditor',
   'ci_composite',
+  'codeeditor',
   'custom_integrations',
-  'url_template_editor',
-  'dashboard',
   'dashboard_enhanced',
+  'dashboard',
   'data_enhanced',
   'embeddable',
   'expression_error',
@@ -31,11 +30,13 @@ const STORYBOOKS = [
   'expression_tagcloud',
   'fleet',
   'infra',
-  'security_solution',
-  'ui_actions_enhanced',
+  'lists',
   'observability',
   'presentation',
-  'lists',
+  'security_solution',
+  'shared_ux',
+  'ui_actions_enhanced',
+  'url_template_editor',
 ];
 
 const GITHUB_CONTEXT = 'Build and Publish Storybooks';

.ci/Dockerfile

@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=16.13.0
+ARG NODE_VERSION=16.13.2
 
 FROM node:${NODE_VERSION} AS base
 

.github/CODEOWNERS

@@ -11,7 +11,7 @@
 /packages/kbn-docs-utils/ @elastic/kibana-tech-leads @elastic/kibana-operations
 
 # Virtual teams
-/x-pack/plugins/rule_registry/ @elastic/rac
+/x-pack/plugins/rule_registry/ @elastic/rac @elastic/response-ops
 
 # Data Discovery
 /src/plugins/discover/ @elastic/kibana-data-discovery
@@ -163,6 +163,7 @@
 /src/plugins/input_control_vis/ @elastic/kibana-presentation
 /src/plugins/vis_type_markdown/ @elastic/kibana-presentation
 /src/plugins/presentation_util/ @elastic/kibana-presentation
+/src/plugins/controls/ @elastic/kibana-presentation
 /test/functional/apps/dashboard/  @elastic/kibana-presentation
 /test/functional/apps/dashboard_elements/  @elastic/kibana-presentation
 /x-pack/plugins/canvas/  @elastic/kibana-presentation
@@ -323,70 +324,48 @@
 /examples/preboot_example/ @elastic/kibana-security @elastic/kibana-core
 #CC# /x-pack/plugins/security/ @elastic/kibana-security
 
-# Kibana Alerting Services
-/x-pack/plugins/alerting/ @elastic/kibana-alerting-services
-/x-pack/plugins/actions/ @elastic/kibana-alerting-services
-/x-pack/plugins/event_log/ @elastic/kibana-alerting-services
-/x-pack/plugins/task_manager/ @elastic/kibana-alerting-services
-/x-pack/test/alerting_api_integration/ @elastic/kibana-alerting-services
-/x-pack/test/plugin_api_integration/test_suites/task_manager/ @elastic/kibana-alerting-services
-/x-pack/plugins/triggers_actions_ui/ @elastic/kibana-alerting-services
-/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/ @elastic/kibana-alerting-services
-/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/ @elastic/kibana-alerting-services
-/docs/user/alerting/ @elastic/kibana-alerting-services
-/docs/management/connectors/ @elastic/kibana-alerting-services
-#CC# /x-pack/plugins/stack_alerts @elastic/kibana-alerting-services
+# Response Ops team
+/x-pack/plugins/alerting/ @elastic/response-ops
+/x-pack/plugins/actions/ @elastic/response-ops
+/x-pack/plugins/event_log/ @elastic/response-ops
+/x-pack/plugins/task_manager/ @elastic/response-ops
+/x-pack/test/alerting_api_integration/ @elastic/response-ops
+/x-pack/test/plugin_api_integration/test_suites/task_manager/ @elastic/response-ops
+/x-pack/plugins/triggers_actions_ui/ @elastic/response-ops
+/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/ @elastic/response-ops
+/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/ @elastic/response-ops
+/docs/user/alerting/ @elastic/response-ops
+/docs/management/connectors/ @elastic/response-ops
+#CC# /x-pack/plugins/stack_alerts @elastic/response-ops
+/x-pack/plugins/cases @elastic/response-ops
+/x-pack/test/cases_api_integration @elastic/response-ops
 
 # Enterprise Search
-# Shared
-/x-pack/plugins/enterprise_search/* @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/common/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/public/* @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/public/applications/* @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/public/applications/enterprise_search/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/public/applications/shared/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/public/applications/__mocks__/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/server/* @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/server/lib/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/server/__mocks__/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/server/collectors/enterprise_search/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/server/collectors/lib/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/server/routes/enterprise_search/ @elastic/enterprise-search-frontend
-/x-pack/plugins/enterprise_search/server/saved_objects/enterprise_search/ @elastic/enterprise-search-frontend
+/x-pack/plugins/enterprise_search @elastic/enterprise-search-frontend
 /x-pack/test/functional_enterprise_search/ @elastic/enterprise-search-frontend
-# App Search
-/x-pack/plugins/enterprise_search/public/applications/app_search/ @elastic/app-search-frontend
-/x-pack/plugins/enterprise_search/server/routes/app_search/ @elastic/app-search-frontend
-/x-pack/plugins/enterprise_search/server/collectors/app_search/ @elastic/app-search-frontend
-/x-pack/plugins/enterprise_search/server/saved_objects/app_search/ @elastic/app-search-frontend
-# Workplace Search
-/x-pack/plugins/enterprise_search/public/applications/workplace_search/ @elastic/workplace-search-frontend
-/x-pack/plugins/enterprise_search/server/routes/workplace_search/ @elastic/workplace-search-frontend
-/x-pack/plugins/enterprise_search/server/collectors/workplace_search/ @elastic/workplace-search-frontend
-/x-pack/plugins/enterprise_search/server/saved_objects/workplace_search/ @elastic/workplace-search-frontend
-
-# Stack Management
-/src/plugins/dev_tools/ @elastic/kibana-stack-management
-/src/plugins/console/  @elastic/kibana-stack-management
-/src/plugins/es_ui_shared/  @elastic/kibana-stack-management
-/src/plugins/management/ @elastic/kibana-stack-management
-/x-pack/plugins/cross_cluster_replication/  @elastic/kibana-stack-management
-/x-pack/plugins/index_lifecycle_management/  @elastic/kibana-stack-management
-/x-pack/plugins/grokdebugger/  @elastic/kibana-stack-management
-/x-pack/plugins/index_management/  @elastic/kibana-stack-management
-/x-pack/plugins/license_api_guard/  @elastic/kibana-stack-management
-/x-pack/plugins/license_management/  @elastic/kibana-stack-management
-/x-pack/plugins/painless_lab/  @elastic/kibana-stack-management
-/x-pack/plugins/remote_clusters/  @elastic/kibana-stack-management
-/x-pack/plugins/rollup/  @elastic/kibana-stack-management
-/x-pack/plugins/searchprofiler/  @elastic/kibana-stack-management
-/x-pack/plugins/snapshot_restore/  @elastic/kibana-stack-management
-/x-pack/plugins/upgrade_assistant/  @elastic/kibana-stack-management
-/x-pack/plugins/watcher/  @elastic/kibana-stack-management
-/x-pack/plugins/ingest_pipelines/  @elastic/kibana-stack-management
-/packages/kbn-ace/ @elastic/kibana-stack-management
-/packages/kbn-monaco/ @elastic/kibana-stack-management
-#CC# /x-pack/plugins/cross_cluster_replication/ @elastic/kibana-stack-management
+
+# Management Experience - Deployment Management
+/src/plugins/dev_tools/ @elastic/platform-deployment-management
+/src/plugins/console/  @elastic/platform-deployment-management
+/src/plugins/es_ui_shared/  @elastic/platform-deployment-management
+/src/plugins/management/ @elastic/platform-deployment-management
+/x-pack/plugins/cross_cluster_replication/  @elastic/platform-deployment-management
+/x-pack/plugins/index_lifecycle_management/  @elastic/platform-deployment-management
+/x-pack/plugins/grokdebugger/  @elastic/platform-deployment-management
+/x-pack/plugins/index_management/  @elastic/platform-deployment-management
+/x-pack/plugins/license_api_guard/  @elastic/platform-deployment-management
+/x-pack/plugins/license_management/  @elastic/platform-deployment-management
+/x-pack/plugins/painless_lab/  @elastic/platform-deployment-management
+/x-pack/plugins/remote_clusters/  @elastic/platform-deployment-management
+/x-pack/plugins/rollup/  @elastic/platform-deployment-management
+/x-pack/plugins/searchprofiler/  @elastic/platform-deployment-management
+/x-pack/plugins/snapshot_restore/  @elastic/platform-deployment-management
+/x-pack/plugins/upgrade_assistant/  @elastic/platform-deployment-management
+/x-pack/plugins/watcher/  @elastic/platform-deployment-management
+/x-pack/plugins/ingest_pipelines/  @elastic/platform-deployment-management
+/packages/kbn-ace/ @elastic/platform-deployment-management
+/packages/kbn-monaco/ @elastic/platform-deployment-management
+#CC# /x-pack/plugins/cross_cluster_replication/ @elastic/platform-deployment-management
 
 # Security Solution
 /x-pack/test/endpoint_api_integration_no_ingest/ @elastic/security-solution
@@ -402,9 +381,7 @@
 #CC# /x-pack/plugins/security_solution/ @elastic/security-solution
 
 # Security Solution sub teams
-/x-pack/plugins/cases @elastic/security-threat-hunting
 /x-pack/plugins/timelines @elastic/security-threat-hunting
-/x-pack/test/cases_api_integration @elastic/security-threat-hunting
 /x-pack/plugins/lists @elastic/security-detections-response
 
 ## Security Solution sub teams - security-onboarding-and-lifecycle-mgt
@@ -476,3 +453,8 @@ x-pack/test/security_solution_cypress @elastic/security-engineering-productivity
 
 # EUI design
 /src/plugins/kibana_react/public/page_template/ @elastic/eui-design @elastic/kibana-app-services
+
+# Application Experience
+
+## Shared UX
+/src/plugins/shared_ux @elastic/shared-ux

.github/workflows/backport.yml

@@ -39,6 +39,3 @@ jobs:
           github_token: ${{secrets.KIBANAMACHINE_TOKEN}}
           commit_user: kibanamachine
           commit_email: [email protected]
-          auto_merge: 'true'
-          auto_merge_method: 'squash'
-          manual_backport_command_template: 'node scripts/backport --pr %pullNumber%'