Merge branch 'main' into fips-agent-test

.buildkite/ftr_configs.yml

@@ -98,7 +98,8 @@ enabled:
   - test/api_integration/config.js
   - test/examples/config.js
   - test/functional/apps/bundles/config.ts
-  - test/functional/apps/console/config.ts
+  - test/functional/apps/console/monaco/config.ts
+  - test/functional/apps/console/ace/config.ts
   - test/functional/apps/context/config.ts
   - test/functional/apps/dashboard_elements/controls/common/config.ts
   - test/functional/apps/dashboard_elements/controls/options_list/config.ts
@@ -115,6 +116,7 @@ enabled:
   - test/functional/apps/discover/ccs_compatibility/config.ts
   - test/functional/apps/discover/classic/config.ts
   - test/functional/apps/discover/embeddable/config.ts
+  - test/functional/apps/discover/esql/config.ts
   - test/functional/apps/discover/group1/config.ts
   - test/functional/apps/discover/group2_data_grid1/config.ts
   - test/functional/apps/discover/group2_data_grid2/config.ts
@@ -125,6 +127,7 @@ enabled:
   - test/functional/apps/discover/group6/config.ts
   - test/functional/apps/discover/group7/config.ts
   - test/functional/apps/discover/group8/config.ts
+  - test/functional/apps/discover/context_awareness/config.ts
   - test/functional/apps/getting_started/config.ts
   - test/functional/apps/home/config.ts
   - test/functional/apps/kibana_overview/config.ts
@@ -246,6 +249,7 @@ enabled:
   - x-pack/test/fleet_api_integration/config.epm.ts
   - x-pack/test/fleet_api_integration/config.fleet.ts
   - x-pack/test/fleet_api_integration/config.package_policy.ts
+  - x-pack/test/fleet_api_integration/config.space_awareness.ts
   - x-pack/test/fleet_functional/config.ts
   - x-pack/test/ftr_apis/security_and_spaces/config.ts
   - x-pack/test/functional_basic/apps/ml/permissions/config.ts
@@ -324,6 +328,7 @@ enabled:
   - x-pack/test/functional/apps/search_playground/config.ts
   - x-pack/test/functional/apps/snapshot_restore/config.ts
   - x-pack/test/functional/apps/spaces/config.ts
+  - x-pack/test/functional/apps/spaces/solution_view_flag_enabled/config.ts
   - x-pack/test/functional/apps/status_page/config.ts
   - x-pack/test/functional/apps/transform/creation/index_pattern/config.ts
   - x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/config.ts
@@ -405,6 +410,7 @@ enabled:
   - x-pack/test/spaces_api_integration/security_and_spaces/config_trial.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_trial.ts
   - x-pack/test/spaces_api_integration/spaces_only/config.ts
+  - x-pack/test/task_manager_claimer_mget/config.ts
   - x-pack/test/ui_capabilities/security_and_spaces/config.ts
   - x-pack/test/ui_capabilities/spaces_only/config.ts
   - x-pack/test/upgrade_assistant_integration/config.js
@@ -421,6 +427,7 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.saved_objects_management.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.context_awareness.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group3.ts
@@ -433,6 +440,7 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
   - x-pack/test_serverless/functional/test_suites/search/config.saved_objects_management.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.context_awareness.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group3.ts
@@ -441,8 +449,10 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/security/config.ts
   - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
-  - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.basic.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.essentials.ts
   - x-pack/test_serverless/functional/test_suites/security/config.saved_objects_management.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.context_awareness.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group2.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group3.ts
@@ -571,4 +581,3 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.integrations.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.integrations_feature_flag.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/integrations_feature_flag.config.ts
-

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -25,7 +25,7 @@ spec:
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 7.17 8.13 8.14
+      branch_configuration: main 7.17 8.14
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge.yml

.buildkite/pipeline-utils/buildkite/client.ts

@@ -282,7 +282,7 @@ export class BuildkiteClient {
         hasRetries = true;
         const isPreemptionFailure =
           job.state === 'failed' &&
-          job.agent?.meta_data?.includes('spot=true') &&
+          job.agent?.meta_data?.some((el) => ['spot=true', 'gcp:preemptible=true'].includes(el)) &&
           job.exit_status === -1;
 
         if (!isPreemptionFailure) {

.buildkite/pipeline-utils/test-failures/annotate.ts

@@ -177,7 +177,7 @@ export const annotateTestFailures = async () => {
     );
   }
 
-  if (process.env.SLACK_NOTIFICATIONS_ENABLED === 'true') {
+  if (process.env.ELASTIC_SLACK_NOTIFICATIONS_ENABLED === 'true') {
     buildkite.setMetadata(
       'slack:test_failures:body',
       getSlackMessage(failures, failureHtmlArtifacts)

.buildkite/pipelines/esql_grammar_sync.yml

@@ -8,9 +8,9 @@ steps:
       provider: gcp
       machineType: n2-standard-2
       preemptible: true
-  - command: .buildkite/scripts/steps/esql_generate_function_definitions.sh
-    label: Generate Function Definitions
-    timeout_in_minutes: 10
+  - command: .buildkite/scripts/steps/esql_generate_function_metadata.sh
+    label: Generate Function Metadata
+    timeout_in_minutes: 15
     agents:
       image: family/kibana-ubuntu-2004
       imageProject: elastic-images-prod

.buildkite/scripts/common/deployment_credentials.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/vault_fns.sh
+
+## Usage
+# ./deployment_credentials.sh set <key-path> <key=value> <key=value> ...
+# ./deployment_credentials.sh unset <key-path>
+# ./deployment_credentials.sh print <key-path>
+
+if [[ "${1:-}" == "set" ]]; then
+  set_in_legacy_vault "${@:2}"
+elif [[ "${1:-}" == "unset" ]]; then
+  unset_in_legacy_vault "${@:2}"
+elif [[ "${1:-}" == "print" ]]; then
+  print_legacy_vault_read "${2}"
+else
+  echo "Unknown command: $1"
+  exit 1
+fi

.buildkite/scripts/common/vault_fns.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
-# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
-if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
+# TODO: rewrite after https://github.com/elastic/kibana-operations/issues/15 is done
+export LEGACY_VAULT_ADDR="https://secrets.elastic.co:8200"
+if [[ "${VAULT_ADDR:-}" == "$LEGACY_VAULT_ADDR" ]]; then
   VAULT_PATH_PREFIX="secret/kibana-issues/dev"
   VAULT_KV_PREFIX="secret/kibana-issues/dev"
   IS_LEGACY_VAULT_ADDR=true
@@ -85,3 +86,56 @@ function get_vault_secret_id() {
 
     echo "$VAULT_SECRET_ID"
 }
+
+function set_in_legacy_vault() {
+  key_path=$1
+  shift
+  fields=("$@")
+
+  VAULT_ROLE_ID="$(get_vault_role_id)"
+  VAULT_SECRET_ID="$(get_vault_secret_id)"
+  VAULT_TOKEN_BAK="$VAULT_TOKEN"
+
+  # Make sure to either keep this variable name `VAULT_TOKEN` or unset `VAULT_TOKEN`,
+  # otherwise the VM's default token will be used, that's connected to the ci-prod vault instance
+  VAULT_TOKEN=$(VAULT_ADDR=$LEGACY_VAULT_ADDR vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+  VAULT_ADDR=$LEGACY_VAULT_ADDR vault login -no-print "$VAULT_TOKEN"
+
+  set +e
+  # shellcheck disable=SC2068
+  vault write -address=$LEGACY_VAULT_ADDR "secret/kibana-issues/dev/cloud-deploy/$key_path" ${fields[@]}
+  EXIT_CODE=$?
+  set -e
+
+  VAULT_TOKEN="$VAULT_TOKEN_BAK"
+
+  return $EXIT_CODE
+}
+
+function unset_in_legacy_vault() {
+  key_path=$1
+
+  VAULT_ROLE_ID="$(get_vault_role_id)"
+  VAULT_SECRET_ID="$(get_vault_secret_id)"
+  VAULT_TOKEN_BAK="$VAULT_TOKEN"
+
+  # Make sure to either keep this variable name `VAULT_TOKEN` or unset `VAULT_TOKEN`,
+  # otherwise the VM's default token will be used, that's connected to the ci-prod vault instance
+  VAULT_TOKEN=$(VAULT_ADDR=$LEGACY_VAULT_ADDR vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+  VAULT_ADDR=$LEGACY_VAULT_ADDR vault login -no-print "$VAULT_TOKEN"
+
+  set +e
+  vault delete -address=$LEGACY_VAULT_ADDR "secret/kibana-issues/dev/cloud-deploy/$key_path"
+  EXIT_CODE=$?
+  set -e
+
+  VAULT_TOKEN="$VAULT_TOKEN_BAK"
+
+  return $EXIT_CODE
+}
+
+function print_legacy_vault_read() {
+  key_path=$1
+
+  echo "vault read -address=$LEGACY_VAULT_ADDR secret/kibana-issues/dev/cloud-deploy/$key_path"
+}

.buildkite/scripts/packer_cache.sh

@@ -17,6 +17,3 @@ done
 for version in $(cat versions.json | jq -r '.versions[].version'); do
   node x-pack/plugins/security_solution/scripts/endpoint/agent_downloader --version "$version"
 done
-
-echo "--- Cloning repos for docs build"
-node scripts/validate_next_docs --clone-only

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -146,15 +146,14 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/fips.yml'));
     }
 
-    if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/build_project.yml'));
-    }
     if (
       GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
       GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||
       GITHUB_PR_LABELS.includes('ci:project-deploy-security')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_project.yml'));
+    } else if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/build_project.yml'));
     }
 
     if (

.buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh

@@ -15,7 +15,11 @@ buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true
 source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh
 
 echo "--- Running test script $1"
-TARGET_SCRIPT=$1 node .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/start_api_ftr_execution
+
+cd x-pack/test/security_solution_api_integration
+set +e
+
+TARGET_SCRIPT=$1 node ./scripts/mki_start_api_ftr_execution
 cmd_status=$?
 echo "Exit code with status: $cmd_status"
 exit $cmd_status

.buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh

@@ -8,6 +8,13 @@ vault_get security-quality-gate/role-users data -format=json > .ftr/role_users.j
 vault_get security-quality-gate/role-users/sec-sol-auto-01 data -format=json > .ftr/sec-sol-auto-01.json
 vault_get security-quality-gate/role-users/sec-sol-auto-02 data -format=json > .ftr/sec-sol-auto-02.json
 vault_get security-quality-gate/role-users/sec-sol-auto-03 data -format=json > .ftr/sec-sol-auto-03.json
+vault_get security-quality-gate/role-users/sec-sol-auto-04 data -format=json > .ftr/sec-sol-auto-04.json
+vault_get security-quality-gate/role-users/sec-sol-auto-05 data -format=json > .ftr/sec-sol-auto-05.json
+vault_get security-quality-gate/role-users/sec-sol-auto-06 data -format=json > .ftr/sec-sol-auto-06.json
+vault_get security-quality-gate/role-users/sec-sol-auto-07 data -format=json > .ftr/sec-sol-auto-07.json
+vault_get security-quality-gate/role-users/sec-sol-auto-08 data -format=json > .ftr/sec-sol-auto-08.json
+vault_get security-quality-gate/role-users/sec-sol-auto-09 data -format=json > .ftr/sec-sol-auto-09.json
+vault_get security-quality-gate/role-users/sec-sol-auto-10 data -format=json > .ftr/sec-sol-auto-10.json
 
 # The vault entries relevant to QA Cloud
 export CLOUD_QA_API_KEY=$(vault_get security-solution-quality-gate qa_api_key)

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -82,7 +82,9 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
 
   echo "Writing to vault..."
 
-  vault_kv_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+  set_in_legacy_vault "$CLOUD_DEPLOYMENT_NAME" \
+    username="$CLOUD_DEPLOYMENT_USERNAME" \
+    password="$CLOUD_DEPLOYMENT_PASSWORD"
 
   echo "Enabling Stack Monitoring..."
   jq '
@@ -114,6 +116,7 @@ else
   ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 fi
 
+VAULT_READ_COMMAND=$(print_legacy_vault_read "$CLOUD_DEPLOYMENT_NAME")
 
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
 CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
@@ -125,9 +128,7 @@ Kibana: $CLOUD_DEPLOYMENT_KIBANA_URL
 
 Elasticsearch: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL
 
-Credentials: \`vault kv get $VAULT_KV_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
-
-(Stored in the production vault: VAULT_ADDR=https://vault-ci-prod.elastic.dev, more info: https://docs.elastic.dev/ci/using-secrets)
+Credentials: \`$VAULT_READ_COMMAND\`
 
 Kibana image: \`$KIBANA_CLOUD_IMAGE\`
 

.buildkite/scripts/steps/cloud/purge_deployments.ts

@@ -7,12 +7,10 @@
  */
 
 import { execSync } from 'child_process';
+import { getKibanaDir } from '#pipeline-utils';
 
 const deploymentsListJson = execSync('ecctl deployment list --output json').toString();
 const { deployments } = JSON.parse(deploymentsListJson);
-const secretBasePath = process.env.VAULT_ADDR?.match(/secrets\.elastic\.co/g)
-  ? 'secret/kibana-issues/dev'
-  : 'secret/ci/elastic-kibana';
 
 const prDeployments = deployments.filter((deployment: any) =>
   deployment.name.startsWith('kibana-pr-')
@@ -70,7 +68,9 @@ for (const deployment of deploymentsToPurge) {
   console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
   try {
     execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
-    execSync(`vault delete ${secretBasePath}/cloud-deploy/${deployment.name}`, {
+
+    execSync(`.buildkite/scripts/common/deployment_credentials.sh unset ${deployment.name}`, {
+      cwd: getKibanaDir(),
       stdio: 'inherit',
     });
   } catch (ex) {

.buildkite/scripts/steps/cloud/purge_projects.ts

@@ -8,6 +8,7 @@
 
 import { execSync } from 'child_process';
 import axios from 'axios';
+import { getKibanaDir } from '#pipeline-utils';
 
 async function getPrProjects() {
   const match = /^(keep.?)?kibana-pr-([0-9]+)-(elasticsearch|security|observability)$/;
@@ -43,12 +44,19 @@ async function getPrProjects() {
 async function deleteProject({
   type,
   id,
+  name,
 }: {
   type: 'elasticsearch' | 'observability' | 'security';
   id: number;
+  name: string;
 }) {
   try {
     await projectRequest.delete(`/api/v1/serverless/projects/${type}/${id}`);
+
+    execSync(`.buildkite/scripts/common/deployment_credentials.sh unset ${name}`, {
+      cwd: getKibanaDir(),
+      stdio: 'inherit',
+    });
   } catch (e) {
     if (e.isAxiosError) {
       const message =
@@ -61,7 +69,7 @@ async function deleteProject({
 
 async function purgeProjects() {
   const prProjects = await getPrProjects();
-  const projectsToPurge = [];
+  const projectsToPurge: typeof prProjects = [];
   for (const project of prProjects) {
     const NOW = new Date().getTime() / 1000;
     const DAY_IN_SECONDS = 60 * 60 * 24;

.buildkite/scripts/steps/code_generation/security_solution_codegen.sh

@@ -6,5 +6,17 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Security Solution OpenAPI Code Generation
 
+echo OpenAPI Common Package
+
+(cd packages/kbn-openapi-common && yarn openapi:generate)
+check_for_changed_files "yarn openapi:generate" true
+
+echo Lists API Common Package
+
+(cd packages/kbn-securitysolution-lists-common && yarn openapi:generate)
+check_for_changed_files "yarn openapi:generate" true
+
+echo Security Solution Plugin
+
 (cd x-pack/plugins/security_solution && yarn openapi:generate)
 check_for_changed_files "yarn openapi:generate" true