Merge branch 'main' into log-flyout-highlights

.buildkite/ftr_configs.yml

@@ -462,6 +462,9 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/ess.config.ts
+
 
 
 

.buildkite/pipelines/es_serverless/emergency_relelease_branch_testing.yml

@@ -0,0 +1,17 @@
+# https://buildkite.com/elastic/kibana-serverless-emergency-release-branch-testing
+
+## Triggers the artifacts container image build for emergency releases
+agents:
+  queue: kibana-default
+
+notify:
+  - slack: "#kibana-mission-control"
+    if: "build.state == 'passed' || build.state == 'failed' || build.state == 'scheduled'"
+
+steps:
+  - trigger: "kibana-artifacts-container-image"
+    label: ":docker: Build Kibana Artifacts Container Image"
+    build:
+      branch: $BUILDKITE_BRANCH
+      commit: $BUILDKITE_COMMIT
+      message: Running PR build for $BUILDKITE_BRANCH

.buildkite/pipelines/security_solution/base.yml → .buildkite/pipelines/security_solution/security_solution_cypress.yml

@@ -1,5 +1,5 @@
 steps:
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless
     label: 'Serverless MKI QA Security Cypress Tests'
     agents:
       queue: n2-4-spot
@@ -11,7 +11,7 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
     label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot
@@ -23,7 +23,7 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
     label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot

.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/pipeline.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "Running the EDR-Workflows testing for Kibana"

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh → .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/pipeline.sh

@@ -2,4 +2,4 @@
 
 set -euo pipefail
 
-ts-node .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts
+ts-node .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/pipeline.ts

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts → .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/pipeline.ts

@@ -28,7 +28,9 @@ const uploadPipeline = (pipelineContent: string | object) => {
   try {
     const pipeline = [];
 
-    pipeline.push(getPipeline('.buildkite/pipelines/security_solution/base.yml', false));
+    pipeline.push(
+      getPipeline('.buildkite/pipelines/security_solution/security_solution_cypress.yml', false)
+    );
     // remove duplicated steps
     uploadPipeline([...new Set(pipeline)].join('\n'));
   } catch (ex) {

.buildkite/scripts/steps/serverless/build_and_deploy.sh

@@ -6,91 +6,102 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 source .buildkite/scripts/steps/artifacts/docker_image.sh
 
-PROJECT_TYPE=""
-is_pr_with_label "ci:project-deploy-elasticsearch" && PROJECT_TYPE="elasticsearch"
-is_pr_with_label "ci:project-deploy-observability" && PROJECT_TYPE="observability"
-is_pr_with_label "ci:project-deploy-security" && PROJECT_TYPE="security"
-if [ -z "${PROJECT_TYPE}" ]; then
-  echo "Mising project type"
-  exit 10
-fi
-
-PROJECT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST-$PROJECT_TYPE"
-PROJECT_CREATE_CONFIGURATION='{
-  "name": "'"$PROJECT_NAME"'",
-  "region_id": "aws-eu-west-1",
-  "overrides": {
-      "kibana": {
-          "docker_image": "'"$KIBANA_IMAGE"'"
-      }
-  }
-}'
-PROJECT_UPDATE_CONFIGURATION='{
-  "name": "'"$PROJECT_NAME"'",
-  "overrides": {
-      "kibana": {
-          "docker_image": "'"$KIBANA_IMAGE"'"
-      }
-  }
-}'
-
-echo "--- Create project"
-DEPLOY_LOGS=$(mktemp --suffix ".json")
-
-echo "Checking if project already exists..."
-curl -s \
-  -H "Authorization: ApiKey $PROJECT_API_KEY" \
-  "${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}" \
-  -XGET &>> $DEPLOY_LOGS
-
-PROJECT_ID=$(jq -r --slurp '[.[0].items[] | select(.name == "'$PROJECT_NAME'")] | .[0].id' $DEPLOY_LOGS)
-if [ -z "${PROJECT_ID}" ] || [ "$PROJECT_ID" = 'null' ]; then
-  echo "Creating project..."
-  curl -s \
-    -H "Authorization: ApiKey $PROJECT_API_KEY" \
-    -H "Content-Type: application/json" \
-    "${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}" \
-    -XPOST -d "$PROJECT_CREATE_CONFIGURATION" &>> $DEPLOY_LOGS
-
-  PROJECT_ID=$(jq -r --slurp '.[1].id' $DEPLOY_LOGS)
 
-  echo "Get credentials..."
-  curl -s -XPOST -H "Authorization: ApiKey $PROJECT_API_KEY" \
-    "${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}/${PROJECT_ID}/_reset-credentials" &>> $DEPLOY_LOGS
-
-  PROJECT_USERNAME=$(jq -r --slurp '.[2].username' $DEPLOY_LOGS)
-  PROJECT_PASSWORD=$(jq -r --slurp '.[2].password' $DEPLOY_LOGS)
-
-  echo "Write to vault..."
-  VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
-  VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
-  VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
-  retry 5 30 vault login -no-print "$VAULT_TOKEN"
-  retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
-else
-  echo "Updating project..."
+deploy() {
+  PROJECT_TYPE=$1
+  case $PROJECT_TYPE in
+    elasticsearch)
+      PROJECT_TYPE_LABEL='Elasticsearch Serverless'
+    ;;
+    observability)
+      PROJECT_TYPE_LABEL='Observability'
+    ;;
+    security)
+      PROJECT_TYPE_LABEL='Security'
+    ;;
+  esac
+
+  PROJECT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST-$PROJECT_TYPE"
+  PROJECT_CREATE_CONFIGURATION='{
+    "name": "'"$PROJECT_NAME"'",
+    "region_id": "aws-eu-west-1",
+    "overrides": {
+        "kibana": {
+            "docker_image": "'"$KIBANA_IMAGE"'"
+        }
+    }
+  }'
+  PROJECT_UPDATE_CONFIGURATION='{
+    "name": "'"$PROJECT_NAME"'",
+    "overrides": {
+        "kibana": {
+            "docker_image": "'"$KIBANA_IMAGE"'"
+        }
+    }
+  }'
+
+  echo "--- Create $PROJECT_TYPE_LABEL project"
+  DEPLOY_LOGS=$(mktemp --suffix ".json")
+
+  echo "Checking if project already exists..."
   curl -s \
     -H "Authorization: ApiKey $PROJECT_API_KEY" \
-    -H "Content-Type: application/json" \
-    "${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}/${PROJECT_ID}" \
-    -XPUT -d "$PROJECT_UPDATE_CONFIGURATION" &>> $DEPLOY_LOGS
-fi
-
-PROJECT_KIBANA_URL=$(jq -r --slurp '.[1].endpoints.kibana' $DEPLOY_LOGS)
-PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
-PROJECT_ELASTICSEARCH_URL=$(jq -r --slurp '.[1].endpoints.elasticsearch' $DEPLOY_LOGS)
-
-cat << EOF | buildkite-agent annotate --style "info" --context project
-  ### Project Deployment
-
-  Kibana: $PROJECT_KIBANA_LOGIN_URL
-
-  Elasticsearch: $PROJECT_ELASTICSEARCH_URL
-
-  Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME\`
-
-  Kibana image: \`$KIBANA_IMAGE\`
+    "${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}" \
+    -XGET &>> $DEPLOY_LOGS
+
+  PROJECT_ID=$(jq -r --slurp '[.[0].items[] | select(.name == "'$PROJECT_NAME'")] | .[0].id' $DEPLOY_LOGS)
+  if [ -z "${PROJECT_ID}" ] || [ "$PROJECT_ID" = 'null' ]; then
+    echo "Creating project..."
+    curl -s \
+      -H "Authorization: ApiKey $PROJECT_API_KEY" \
+      -H "Content-Type: application/json" \
+      "${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}" \
+      -XPOST -d "$PROJECT_CREATE_CONFIGURATION" &>> $DEPLOY_LOGS
+
+    PROJECT_ID=$(jq -r --slurp '.[1].id' $DEPLOY_LOGS)
+
+    echo "Get credentials..."
+    curl -s -XPOST -H "Authorization: ApiKey $PROJECT_API_KEY" \
+      "${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}/${PROJECT_ID}/_reset-credentials" &>> $DEPLOY_LOGS
+
+    PROJECT_USERNAME=$(jq -r --slurp '.[2].username' $DEPLOY_LOGS)
+    PROJECT_PASSWORD=$(jq -r --slurp '.[2].password' $DEPLOY_LOGS)
+
+    echo "Write to vault..."
+    VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+    VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+    VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+    retry 5 30 vault login -no-print "$VAULT_TOKEN"
+    retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME" username="$PROJECT_USERNAME" password="$PROJECT_PASSWORD" id="$PROJECT_ID"
+  else
+    echo "Updating project..."
+    curl -s \
+      -H "Authorization: ApiKey $PROJECT_API_KEY" \
+      -H "Content-Type: application/json" \
+      "${PROJECT_API_DOMAIN}/api/v1/serverless/projects/${PROJECT_TYPE}/${PROJECT_ID}" \
+      -XPUT -d "$PROJECT_UPDATE_CONFIGURATION" &>> $DEPLOY_LOGS
+  fi
+
+  PROJECT_KIBANA_URL=$(jq -r --slurp '.[1].endpoints.kibana' $DEPLOY_LOGS)
+  PROJECT_KIBANA_LOGIN_URL="${PROJECT_KIBANA_URL}/login"
+  PROJECT_ELASTICSEARCH_URL=$(jq -r --slurp '.[1].endpoints.elasticsearch' $DEPLOY_LOGS)
+
+  cat << EOF | buildkite-agent annotate --style "info" --context "project-$PROJECT_TYPE"
+### $PROJECT_TYPE_LABEL Deployment
+
+Kibana: $PROJECT_KIBANA_LOGIN_URL
+
+Elasticsearch: $PROJECT_ELASTICSEARCH_URL
+
+Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$PROJECT_NAME\`
+
+Kibana image: \`$KIBANA_IMAGE\`
 EOF
 
-buildkite-agent meta-data set pr_comment:deploy_project:head "* [Project Deployment](${PROJECT_KIBANA_LOGIN_URL})"
-buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
+  buildkite-agent meta-data set "pr_comment:deploy_project_$PROJECT_TYPE:head" "* [$PROJECT_TYPE_LABEL Deployment](${PROJECT_KIBANA_LOGIN_URL})"
+  buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
+}
+
+is_pr_with_label "ci:project-deploy-elasticsearch" && deploy "elasticsearch"
+is_pr_with_label "ci:project-deploy-observability" && deploy "observability"
+is_pr_with_label "ci:project-deploy-security" && deploy "security"