Merge branch 'main' into csp_readme

.buildkite/disabled_jest_configs.json

@@ -1,5 +1,4 @@
 [
-  "x-pack/plugins/triggers_actions_ui/jest.config.js",
   "x-pack/plugins/watcher/jest.config.js",
   "src/core/server/integration_tests/ui_settings/jest.integration.config.js"
 ]

.buildkite/ftr_configs.yml

@@ -76,7 +76,6 @@ enabled:
   - test/functional/apps/dashboard/group3/config.ts
   - test/functional/apps/dashboard/group4/config.ts
   - test/functional/apps/dashboard/group5/config.ts
-  - test/functional/apps/dashboard/group6/config.ts
   - test/functional/apps/discover/ccs_compatibility/config.ts
   - test/functional/apps/discover/classic/config.ts
   - test/functional/apps/discover/embeddable/config.ts
@@ -97,6 +96,7 @@ enabled:
   - test/functional/apps/visualize/replaced_vislib_chart_types/config.ts
   - test/functional/config.ccs.ts
   - test/functional/config.firefox.js
+  - test/health_gateway/config.ts
   - test/interactive_setup_api_integration/enrollment_flow.config.ts
   - test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts
   - test/interactive_setup_api_integration/manual_configuration_flow.config.ts
@@ -123,6 +123,7 @@ enabled:
   - x-pack/test/alerting_api_integration/basic/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
   - x-pack/test/alerting_api_integration/spaces_only/config.ts
   - x-pack/test/api_integration_basic/config.ts
@@ -185,6 +186,7 @@ enabled:
   - x-pack/test/functional/apps/lens/group3/config.ts
   - x-pack/test/functional/apps/lens/open_in_lens/tsvb/config.ts
   - x-pack/test/functional/apps/lens/open_in_lens/agg_based/config.ts
+  - x-pack/test/functional/apps/lens/open_in_lens/dashboard/config.ts
   - x-pack/test/functional/apps/license_management/config.ts
   - x-pack/test/functional/apps/logstash/config.ts
   - x-pack/test/functional/apps/management/config.ts

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -166,7 +166,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${CYPRESS_SUITE}.sh`,
         label: group.name,
-        agents: { queue: 'ci-group-6' },
+        agents: { queue: 'n2-4-spot' },
         depends_on: 'build',
         parallelism: testSuite.count,
         concurrency,

.buildkite/pipelines/performance/daily.yml

@@ -27,12 +27,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - label: '🚢 Performance Tests dataset extraction for scalability benchmarking'
-    command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
-    agents:
-      queue: n2-2
-    depends_on: tests
-
   - label: '📈 Report performance metrics to ci-stats'
     command: .buildkite/scripts/steps/functional/report_performance_metrics.sh
     agents:

.buildkite/pipelines/performance/data_set_extraction_daily.yml

@@ -0,0 +1,43 @@
+steps:
+  - label: ':male-mechanic::skin-tone-2: Pre-Build'
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+    agents:
+      queue: kibana-default
+    timeout_in_minutes: 10
+
+  - wait
+
+  - label: ':building_construction: Build Kibana Distribution and Plugins'
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      queue: c2-16
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - label: ':kibana: Performance Tests with Playwright config'
+    command: .buildkite/scripts/steps/functional/performance_playwright.sh
+    agents:
+      queue: n2-2-spot
+    depends_on: build
+    key: tests
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - label: ':ship: Single user journeys dataset extraction for scalability benchmarking'
+    command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
+    agents:
+      queue: n2-2
+    depends_on: tests
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ':male_superhero::skin-tone-2: Post-Build'
+    command: .buildkite/scripts/lifecycle/post_build.sh
+    agents:
+      queue: kibana-default

.buildkite/pipelines/pull_request/apm_cypress.yml

@@ -2,10 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/apm_cypress.sh
     label: 'APM Cypress Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/fleet_cypress.yml

@@ -2,10 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/fleet_cypress.sh
     label: 'Fleet Cypress Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/kbn_handlebars.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/test/kbn_handlebars.sh
+    label: 'Check @kbn/handlebars for upstream differences'
+    agents:
+      queue: n2-2-spot
+    depends_on: build
+    timeout_in_minutes: 5
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/observability_plugin.yml

@@ -2,10 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/observability_plugin.sh
     label: 'Observability @elastic/synthetics Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/osquery_cypress.yml

@@ -2,11 +2,13 @@ steps:
   - command: .buildkite/scripts/steps/functional/osquery_cypress.sh
     label: 'Osquery Cypress Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
     artifact_paths:

.buildkite/pipelines/pull_request/response_ops.yml

@@ -2,10 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/response_ops.sh
     label: 'Rules, Alerts and Exceptions ResponseOps Cypress Tests on Security Solution'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/response_ops_cases.yml

@@ -2,10 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/response_ops_cases.sh
     label: 'Cases Cypress Tests on Security Solution'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/security_solution.yml

@@ -2,11 +2,13 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     parallelism: 4
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/synthetics_plugin.yml

@@ -2,10 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/synthetics_plugin.sh
     label: 'Synthetics @elastic/synthetics Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/threat_intelligence.yml

@@ -2,11 +2,13 @@ steps:
   - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
     label: 'Threat Intelligence Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     parallelism: 4
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/pipelines/pull_request/ux_plugin_e2e.yml

@@ -2,10 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/ux_synthetics_e2e.sh
     label: 'UX Plugin @elastic/synthetics Tests'
     agents:
-      queue: ci-group-6
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -54,6 +54,10 @@ const uploadPipeline = (pipelineContent: string | object) => {
 
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false));
 
+    if (await doAnyChangesMatch([/^packages\/kbn-handlebars/])) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/kbn_handlebars.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([
         /^packages\/kbn-securitysolution-.*/,
@@ -63,6 +67,7 @@ const uploadPipeline = (pipelineContent: string | object) => {
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
         /^x-pack\/test\/security_solution_cypress/,
+        /^fleet_packages\.json/, // It contains reference to prebuilt detection rules, we want to run security solution tests if it changes
       ])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {

.buildkite/scripts/steps/functional/performance_playwright.sh

@@ -12,8 +12,16 @@ is_test_execution_step
 rm -rf "$KIBANA_BUILD_LOCATION"
 .buildkite/scripts/download_build_artifacts.sh
 
-echo "--- Running performance tests"
-node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+if [ "$BUILDKITE_PIPELINE_SLUG" == "kibana-performance-data-set-extraction" ]; then
+  # 'performance-data-set-extraction' uses 'n2-2-spot' agent, performance metrics don't matter
+  # and we skip warmup phase for each test
+  echo "--- Running single user journeys"
+  node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --skip-warmup
+else
+  # pipeline should use bare metal static worker
+  echo "--- Running performance tests"
+  node scripts/run_performance.js --kibana-install-dir "$KIBANA_BUILD_LOCATION"
+fi
 
 echo "--- Upload journey step screenshots"
 JOURNEY_SCREENSHOTS_DIR="${KIBANA_DIR}/data/journey_screenshots"

.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

@@ -46,7 +46,7 @@ cd "${OUTPUT_DIR}/.."
 gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
 cd -
 
-if [ "$BUILDKITE_PIPELINE_SLUG" == "kibana-single-user-performance" ]; then
+if [ "$BUILDKITE_PIPELINE_SLUG" == "kibana-performance-data-set-extraction" ]; then
   echo "--- Promoting '${BUILD_ID}' dataset to LATEST"
   cd "${OUTPUT_DIR}/.."
   echo "${BUILD_ID}" > latest

.buildkite/scripts/steps/scalability/benchmarking.sh

@@ -74,53 +74,9 @@ download_artifacts
 echo "--- Clone kibana-load-testing repo and compile project"
 checkout_and_compile_load_runner
 
-echo "--- Run Scalability Tests with Elasticsearch started only once and Kibana restart before each journey"
+echo "--- Run Scalability Tests"
 cd "$KIBANA_DIR"
-node scripts/es snapshot&
-
-esPid=$!
-# Set trap on EXIT to stop Elasticsearch process
-trap "kill -9 $esPid" EXIT
-
-# unset env vars defined in other parts of CI for automatic APM collection of
-# Kibana. We manage APM config in our FTR config and performance service, and
-# APM treats config in the ENV with a very high precedence.
-unset ELASTIC_APM_ENVIRONMENT
-unset ELASTIC_APM_TRANSACTION_SAMPLE_RATE
-unset ELASTIC_APM_SERVER_URL
-unset ELASTIC_APM_SECRET_TOKEN
-unset ELASTIC_APM_ACTIVE
-unset ELASTIC_APM_CONTEXT_PROPAGATION_ONLY
-unset ELASTIC_APM_GLOBAL_LABELS
-unset ELASTIC_APM_MAX_QUEUE_SIZE
-unset ELASTIC_APM_METRICS_INTERVAL
-unset ELASTIC_APM_CAPTURE_SPAN_STACK_TRACES
-unset ELASTIC_APM_BREAKDOWN_METRICS
-
-
-export TEST_ES_DISABLE_STARTUP=true
-ES_HOST="localhost:9200"
-export TEST_ES_URL="http://elastic:changeme@${ES_HOST}"
-# Overriding Gatling default configuration
-export ES_URL="http://${ES_HOST}"
-
-# Pings the ES server every second for 2 mins until its status is green
-curl --retry 120 \
-  --retry-delay 1 \
-  --retry-connrefused \
-  -I -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow"
-
-export ELASTIC_APM_ACTIVE=true
-
-for journey in scalability_traces/server/*; do
-    export SCALABILITY_JOURNEY_PATH="$KIBANA_DIR/$journey"
-    echo "--- Run scalability file: $SCALABILITY_JOURNEY_PATH"
-    node scripts/functional_tests \
-      --config x-pack/test/scalability/config.ts \
-      --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-      --logToFile \
-      --debug
-done
+node scripts/run_scalability --kibana-install-dir "$KIBANA_BUILD_LOCATION" --journey-path "scalability_traces/server"
 
 echo "--- Upload test results"
 upload_test_results

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -26,6 +26,13 @@ echo "--- downloading jest test run order"
 download_artifact jest_run_order.json .
 configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
 
+echo "+++ ⚠️ WARNING ⚠️"
+echo "
+  console.log(), console.warn(), and console.error() output in jest tests causes a massive amount
+  of noise on CI without any percevable benefit, so they have been disabled. If you want to log
+  output in your test temporarily, you can modify 'packages/kbn-test/src/jest/setup/disable_console_logs.js'
+"
+
 while read -r config; do
   echo "--- $ node scripts/jest --config $config"
 

.buildkite/scripts/steps/test/kbn_handlebars.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo '--- Checking for @kbn/handlebars test changes'
+packages/kbn-handlebars/scripts/check_for_test_changes.sh

.eslintignore

@@ -39,7 +39,7 @@ snapshots.js
 /packages/kbn-test/src/functional_test_runner/lib/config/__tests__/fixtures/
 /packages/kbn-ui-framework/dist
 /packages/kbn-flot-charts/lib
-/packages/kbn-monaco/src/painless/antlr
+/packages/kbn-monaco/src/**/antlr
 
 # Bazel
 /bazel-*