Merge remote-tracking branch 'upstream/main' into eql-sequence-suppre…

…ssion

.buildkite/ftr_oblt_serverless_configs.yml

@@ -6,9 +6,6 @@ disabled:
   - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts
   - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts
 
-  # Serverless feature flag config files (move to enabled after tests are added)
-  - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts
-
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/observability/config.ts
@@ -17,6 +14,7 @@ enabled:
   - x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.saved_objects_management.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.context_awareness.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group1.ts

.buildkite/ftr_search_serverless_configs.yml

@@ -4,9 +4,11 @@ disabled:
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/search/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts
   - x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/search/config.ts
   - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts
   - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
   - x-pack/test_serverless/functional/test_suites/search/config.saved_objects_management.ts
   - x-pack/test_serverless/functional/test_suites/search/config.context_awareness.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -17,21 +17,16 @@ disabled:
   - x-pack/test_serverless/functional/config.base.ts
   - x-pack/test_serverless/shared/config.base.ts
 
-  # Serverless feature flag config files (move to enabled after tests are added)
-  - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts
-  - x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts
-  - x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts
-  - x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts
-  - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
-
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/security/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts
   - x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts
   - x-pack/test_serverless/api_integration/test_suites/security/fleet/config.ts
   - x-pack/test_serverless/functional/test_suites/security/config.screenshots.ts
   - x-pack/test_serverless/functional/test_suites/security/config.ts
   - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.basic.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.essentials.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless.ts
@@ -65,6 +60,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/large_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/serverless.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -47,6 +47,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/large_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/ess.config.ts

.buildkite/pipeline-utils/buildkite/client.test.ts

@@ -24,9 +24,7 @@ describe('BuildkiteClient', () => {
         id: 'id-1',
         retried_in_job_id: 'id-2',
         state: 'failed',
-        agent: {
-          meta_data: ['spot=true'],
-        },
+        agent_query_rules: ['preemptible=true'],
         retried: true,
         exit_status: -1,
         type: 'script',
@@ -35,9 +33,7 @@ describe('BuildkiteClient', () => {
       const retry: Job = {
         id: 'id-2',
         state: 'passed',
-        agent: {
-          meta_data: ['spot=true'],
-        },
+        agent_query_rules: ['preemptible=true'],
         type: 'script',
       } as Job;
 
@@ -58,9 +54,7 @@ describe('BuildkiteClient', () => {
         id: 'id-1',
         retried_in_job_id: 'id-2',
         state: 'failed',
-        agent: {
-          meta_data: ['spot=true'],
-        },
+        agent_query_rules: ['preemptible=true'],
         retried: true,
         exit_status: 1,
         type: 'script',
@@ -69,9 +63,7 @@ describe('BuildkiteClient', () => {
       const retry: Job = {
         id: 'id-2',
         state: 'passed',
-        agent: {
-          meta_data: ['spot=true'],
-        },
+        agent_query_rules: ['preemptible=true'],
         type: 'script',
       } as Job;
 

.buildkite/pipeline-utils/buildkite/client.ts

@@ -285,7 +285,7 @@ export class BuildkiteClient {
         hasRetries = true;
         const isPreemptionFailure =
           job.state === 'failed' &&
-          job.agent?.meta_data?.some((el) => ['spot=true', 'gcp:preemptible=true'].includes(el)) &&
+          job.agent_query_rules?.includes('preemptible=true') &&
           job.exit_status === -1;
 
         if (!isPreemptionFailure) {

.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml

@@ -14,13 +14,13 @@ steps:
         RETRY_TESTS_ON_FAIL: "true"
       message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
 
-  # TODO: Uncomment this code when the integration is ready.
-  # - label: ":pipeline::female-detective::seedling: Trigger Security Solution quality gate script"
-  #   trigger: security-serverless-quality-gate # https://buildkite.com/elastic/security-serverless-quality-gate
-  #   build:
-  #     env:
-  #       ENVIRONMENT: ${ENVIRONMENT}
-  #     message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
+  - label: ":pipeline::female-detective::seedling: Trigger Security Solution quality gate script"
+    trigger: security-serverless-quality-gate # https://buildkite.com/elastic/security-serverless-quality-gate
+    soft_fail: true
+    build:
+      env:
+        ENVIRONMENT: ${ENVIRONMENT}
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
 
   - label: ":ship: Fleet serverless smoke tests for ${ENVIRONMENT}"
     trigger: fleet-smoke-tests # https://buildkite.com/elastic/fleet-smoke-tests

.buildkite/scripts/common/setup_job_env.sh

@@ -8,6 +8,8 @@ if [[ "$(type -t vault_get)" != "function" ]]; then
   source .buildkite/scripts/common/vault_fns.sh
 fi
 
+source .buildkite/scripts/common/util.sh
+
 # Set up general-purpose tokens and credentials
 {
   BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
@@ -18,9 +20,12 @@ fi
 
   KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
   KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
-  if (command -v docker && docker version) &> /dev/null; then
-    echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-  fi
+  function docker_login() {
+      if (command -v docker && docker version) &> /dev/null; then
+        echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+      fi
+  }
+  retry 5 15 docker_login
 }
 
 # Set up a custom ES Snapshot Manifest if one has been specified for this build

.buildkite/scripts/common/util.sh

@@ -178,3 +178,32 @@ print_if_dry_run() {
     echo "DRY_RUN is enabled."
   fi
 }
+
+docker_with_retry () {
+  cmd=$1
+  shift
+  args=("$@")
+  attempt=0
+  max_retries=5
+  sleep_time=15
+
+  while true
+  do
+    attempt=$((attempt+1))
+
+    if [ $attempt -gt $max_retries ]
+    then
+      echo "Docker $cmd retries exceeded, aborting."
+      exit 1
+    fi
+
+    if docker "$cmd" "${args[@]}"
+    then
+      echo "Docker $cmd successful."
+      break
+    else
+      echo "Docker $cmd unsuccessful, attempt '$attempt'... Retrying in $sleep_time"
+      sleep $sleep_time
+    fi
+  done
+}

.buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh

@@ -8,6 +8,7 @@ fi
 .buildkite/scripts/bootstrap.sh
 
 source .buildkite/scripts/steps/artifacts/env.sh
+source .buildkite/scripts/common/util.sh
 
 GIT_ABBREV_COMMIT=${BUILDKITE_COMMIT:0:12}
 KIBANA_IMAGE_TAG="sec-sol-qg-$GIT_ABBREV_COMMIT"
@@ -23,7 +24,7 @@ if docker manifest inspect $KIBANA_IMAGE &> /dev/null; then
   exit 0
 fi
 
-docker pull $KIBANA_BASE_IMAGE:latest
+docker_with_retry pull $KIBANA_BASE_IMAGE:latest
 
 echo "--- Build images"
 node scripts/build \
@@ -50,8 +51,8 @@ docker load < "target/kibana-serverless-$BASE_VERSION-docker-image-aarch64.tar.g
 docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-arm64"
 
 echo "--- Push images"
-docker image push "$KIBANA_IMAGE-arm64"
-docker image push "$KIBANA_IMAGE-amd64"
+docker_with_retry push "$KIBANA_IMAGE-arm64"
+docker_with_retry push "$KIBANA_IMAGE-amd64"
 
 echo "--- Create and push manifests"
 docker manifest create \

.buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+source .buildkite/scripts/common/util.sh
+
 if [ "$KIBANA_MKI_QUALITY_GATE" == "1" ]; then
     echo "Triggered by quality gate!"
     triggered_by="Serverless Quality Gate."
@@ -17,11 +19,11 @@ else
     KBN_IMAGE=${KIBANA_LATEST}
 fi
 
-docker pull ${KBN_IMAGE}
+docker_with_retry pull ${KBN_IMAGE}
 build_date=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.build-date"')
 vcs_ref=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.vcs-ref"')
 vcs_url=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.vcs-url"')
-version=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.version"')   
+version=$(docker inspect ${KBN_IMAGE} | jq -r '.[0].Config.Labels."org.label-schema.version"')
 
 markdown_text="""
 #### $triggered_by
@@ -32,9 +34,9 @@ markdown_text="""
 ---
 
 #### Kibana Container Metadata
-- Build Date            : $build_date 
-- Github Commit Hash    : $vcs_ref 
-- Github Repo           : $vcs_url 
+- Build Date            : $build_date
+- Github Commit Hash    : $vcs_ref
+- Github Repo           : $vcs_url
 - Version               : $version
 """
-echo "${markdown_text//[*\\_]/\\&}" | buildkite-agent annotate --style "info"
+echo "${markdown_text//[*\\_]/\\&}" | buildkite-agent annotate --style "info"

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -28,7 +28,7 @@ docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE"
 if  docker manifest inspect $KIBANA_TEST_IMAGE &> /dev/null; then
   echo "Cloud image already exists, skipping docker push"
 else
-  docker image push "$KIBANA_TEST_IMAGE"
+  docker_with_retry push "$KIBANA_TEST_IMAGE"
 fi
 
 echo "--- Create deployment"

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -6,6 +6,8 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/artifacts/env.sh
 
+source .buildkite/scripts/common/util.sh
+
 GIT_ABBREV_COMMIT=${BUILDKITE_COMMIT:0:12}
 if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
   KIBANA_IMAGE_TAG="git-$GIT_ABBREV_COMMIT"
@@ -50,8 +52,8 @@ if [[ "$SKIP_BUILD" == "false" ]]; then
   docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-arm64"
 
   echo "--- Push images"
-  docker image push "$KIBANA_IMAGE-arm64"
-  docker image push "$KIBANA_IMAGE-amd64"
+  docker_with_retry push "$KIBANA_IMAGE-arm64"
+  docker_with_retry push "$KIBANA_IMAGE-amd64"
 
   echo "--- Create and push manifests"
   docker manifest create \

.buildkite/scripts/steps/artifacts/publish.sh

@@ -54,7 +54,7 @@ chmod -R a+r target/*
 chmod -R a+w target
 
 echo "--- Pull latest Release Manager CLI"
-docker pull docker.elastic.co/infra/release-manager:latest
+docker_with_retry pull docker.elastic.co/infra/release-manager:latest
 
 echo "--- Publish artifacts"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] || [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then

.buildkite/scripts/steps/capture_oas_snapshot.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Capture OAS snapshot
-cmd="node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules"
+cmd="node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules --include-path /api/actions"
 if is_pr && ! is_auto_commit_disabled; then
   cmd="$cmd --update"
 fi

.buildkite/scripts/steps/demo_env/es_and_init.sh

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source "$(dirname "${0}")/config.sh"
+source "$(dirname "${0}")/../../common/util.sh"
 
 "$(dirname "${0}")/auth.sh"
 
@@ -16,7 +17,7 @@ DOCKER_EXPORT_URL=$(curl https://storage.googleapis.com/kibana-ci-es-snapshots-d
 curl "$DOCKER_EXPORT_URL" > target/elasticsearch-docker.tar.gz
 docker load < target/elasticsearch-docker.tar.gz
 docker tag "docker.elastic.co/elasticsearch/elasticsearch:$DEPLOYMENT_VERSION-SNAPSHOT" "$ES_IMAGE"
-docker push "$ES_IMAGE"
+docker_with_retry push "$ES_IMAGE"
 
 echo '--- Prepare yaml'
 

.buildkite/scripts/steps/demo_env/kibana.sh

@@ -4,6 +4,8 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
+source .buildkite/scripts/common/util.sh
+
 source "$(dirname "${0}")/config.sh"
 
 export KIBANA_IMAGE="gcr.io/elastic-kibana-184716/demo/kibana:$DEPLOYMENT_NAME-$(git rev-parse HEAD)"
@@ -15,7 +17,7 @@ echo '--- Build Docker image with example plugins'
 cd target/example_plugins
 BUILT_IMAGE="docker.elastic.co/kibana/kibana:$DEPLOYMENT_VERSION-SNAPSHOT"
 docker build --build-arg BASE_IMAGE="$BUILT_IMAGE" -t "$KIBANA_IMAGE" -f "$KIBANA_DIR/.buildkite/scripts/steps/demo_env/Dockerfile" .
-docker push "$KIBANA_IMAGE"
+docker_with_retry push "$KIBANA_IMAGE"
 cd -
 
 "$(dirname "${0}")/auth.sh"

.buildkite/scripts/steps/es_serverless/annotate_runtime_parameters.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+source .buildkite/scripts/common/util.sh
+
 KIBANA_GITHUB_URL="https://github.com/elastic/kibana"
 ES_SERVERLESS_GITHUB_URL="https://github.com/elastic/elasticsearch-serverless"
 
@@ -15,7 +17,7 @@ fi
 
 # Pull the target image
 if [[ $ES_SERVERLESS_IMAGE != *":git-"* ]]; then
-  docker pull "$ES_SERVERLESS_IMAGE"
+  docker_with_retry pull "$ES_SERVERLESS_IMAGE"
   ES_SERVERLESS_VERSION=$(docker inspect --format='{{json .Config.Labels}}' "$ES_SERVERLESS_IMAGE" | jq -r '.["org.opencontainers.image.revision"]' | cut -c1-12)
 
   IMAGE_WITHOUT_TAG=$(echo "$ES_SERVERLESS_IMAGE" | cut -d: -f1)