Merge branch 'main' into dashboard-listing

.buildkite/ftr_configs.yml

@@ -28,7 +28,6 @@ disabled:
   - x-pack/test/defend_workflows_cypress/cli_config.ts
   - x-pack/test/defend_workflows_cypress/config.ts
   - x-pack/test/defend_workflows_cypress/endpoint_config.ts
-  - x-pack/test/defend_workflows_cypress/visual_config.ts
   - x-pack/plugins/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_onboarding/e2e/ftr_config.ts
@@ -49,6 +48,8 @@ disabled:
   - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts
+  - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts
   - x-pack/plugins/profiling/e2e/ftr_config_open.ts
   - x-pack/plugins/profiling/e2e/ftr_config_runner.ts
   - x-pack/plugins/profiling/e2e/ftr_config.ts
@@ -232,6 +233,9 @@ enabled:
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group9/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/prebuilt_rules/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/bundled_prebuilt_rules_package/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/large_prebuilt_rules_package/config.ts
   - x-pack/test/encrypted_saved_objects_api_integration/config.ts
   - x-pack/test/examples/config.ts
   - x-pack/test/fleet_api_integration/config.agent.ts

.buildkite/pipelines/pull_request/defend_workflows.yml

@@ -14,3 +14,19 @@ steps:
           limit: 1
     artifact_paths:
       - "target/kibana-security-solution/**/*"
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows_vagrant.sh
+    label: 'Defend Workflows Endpoint Cypress Tests'
+    agents:
+      queue: n2-16-virt
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+    artifact_paths:
+      - "target/kibana-security-solution/**/*"

.buildkite/pipelines/serverless.yml

@@ -21,19 +21,6 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: SERVERLESS_ENVIRONMENT=common .buildkite/scripts/steps/functional/serverless_ftr.sh
-    label: 'Serverless Common Tests'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 40
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: SERVERLESS_ENVIRONMENT=observability .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Observability Tests'
     agents:

.buildkite/scripts/build_kibana.sh

@@ -41,6 +41,32 @@ if is_pr_with_label "ci:build-cloud-image"; then
 EOF
 fi
 
+if is_pr_with_label "ci:build-serverless-image"; then
+  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+  GIT_ABBREV_COMMIT=${BUILDKITE_COMMIT:0:12}
+  node scripts/build \
+  --skip-initialize \
+  --skip-generic-folders \
+  --skip-platform-folders \
+  --skip-archives \
+  --docker-images \
+  --docker-namespace="kibana-ci" \
+  --docker-tag="pr-$BUILDKITE_PULL_REQUEST-$GIT_ABBREV_COMMIT" \
+  --docker-push \
+  --skip-docker-ubi \
+  --skip-docker-ubuntu \
+  --skip-docker-cloud \
+  --skip-docker-contexts
+  docker logout docker.elastic.co
+
+  SERVERLESS_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-serverless)
+  buildkite-agent meta-data set pr_comment:deploy_cloud:head "* Kibana Serverless Image: \`$SERVERLESS_IMAGE\`"
+  cat << EOF | buildkite-agent annotate --style "info" --context kibana-serverless-image
+
+  Kibana Serverless Image: \`$SERVERLESS_IMAGE\`
+EOF
+fi
+
 echo "--- Archive Kibana Distribution"
 linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
 installDir="$KIBANA_DIR/install/kibana"

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -60,11 +60,13 @@ const uploadPipeline = (pipelineContent: string | object) => {
 
     if (
       (await doAnyChangesMatch([
+        /^src\/plugins\/controls/,
         /^packages\/kbn-securitysolution-.*/,
         /^x-pack\/plugins\/lists/,
         /^x-pack\/plugins\/security_solution/,
         /^x-pack\/plugins\/timelines/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
+        /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/alerts_table/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
         /^x-pack\/test\/defend_workflows_cypress/,
         /^x-pack\/test\/security_solution_cypress/,

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -83,17 +83,15 @@ echo "--- Trigger image tag update"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
   cat << EOF | buildkite-agent pipeline upload
 steps:
-  - trigger: serverless-gitops-update-stack-image-tag
+  - label: ":argo: Update kibana image tag for kibana-controller using gpctl"
     async: true
-    label: ":argo: Update image tag for Kibana"
     branches: main
+    trigger: gpctl-promote
     build:
       env:
-        IMAGE_TAG: "git-$GIT_ABBREV_COMMIT"
-        SERVICE: kibana-controller
-        NAMESPACE: kibana-ci
-        IMAGE_NAME: kibana-serverless
-        COMMIT_MESSAGE: "gitops: update kibana tag to elastic/kibana@$GIT_ABBREV_COMMIT"
+        SERVICE_COMMIT_HASH: "$GIT_ABBREV_COMMIT"
+        REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/config.yaml
+
 EOF
 
 else

.buildkite/scripts/steps/functional/apm_cypress.sh

@@ -11,9 +11,16 @@ APM_CYPRESS_RECORD_KEY="$(retry 5 5 vault read -field=CYPRESS_RECORD_KEY secret/
 
 export JOB=kibana-apm-cypress
 IS_FLAKY_TEST_RUNNER=${CLI_COUNT:-0}
+GH_APM_TEAM_LABEL="Team:APM"
 
-#Enabling cypress dashboard recording when PR is labeled with `apm:cypress-record` and we are not using the flaky test runner
-if [[ "$IS_FLAKY_TEST_RUNNER" -ne 1 ]] && is_pr_with_label "apm:cypress-record"; then
+if (! is_pr); then
+  echo "--- Add GH labels to buildkite metadata"
+  ts-node .buildkite/scripts/steps/add_gh_labels_to_bk_metadata.ts BUILDKITE_MESSAGE true
+  GH_ON_MERGE_LABELS="$(buildkite-agent meta-data get gh_labels)"
+fi
+
+# Enabling cypress dashboard recording when PR is labeled with `apm:cypress-record` and we are not using the flaky test runner OR on merge with Team:APM label applied
+if ([[ "$IS_FLAKY_TEST_RUNNER" -ne 1 ]] && is_pr_with_label "apm:cypress-record") || ([[ $GH_ON_MERGE_LABELS == *"$GH_APM_TEAM_LABEL"* ]]); then
   CYPRESS_ARGS="--record --key "$APM_CYPRESS_RECORD_KEY" --parallel --ci-build-id "${BUILDKITE_BUILD_ID}""
 else
   CYPRESS_ARGS=""

.buildkite/scripts/steps/functional/common_cypress.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+Xvfb -screen 0 1680x946x24 :99 &
+export DISPLAY=:99

.buildkite/scripts/steps/functional/defend_workflows.sh

@@ -3,14 +3,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-defend-workflows-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-Xvfb -screen 0 1680x946x24 :99 &
-
-export DISPLAY=:99
-
 echo "--- Defend Workflows Cypress tests"
 
 yarn --cwd x-pack/plugins/security_solution cypress:dw:run

.buildkite/scripts/steps/functional/defend_workflows_vagrant.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-defend-workflows-endpoint-cypress
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+
+echo "--- Defend Workflows Endpoint Cypress tests"
+
+yarn --cwd x-pack/plugins/security_solution cypress:dw:endpoint:run

.buildkite/scripts/steps/functional/osquery_cypress.sh

@@ -3,14 +3,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 .buildkite/scripts/bootstrap.sh
 node scripts/build_kibana_platform_plugins.js
 
-Xvfb :99 -screen 0 1600x1200x24 &
-
-export DISPLAY=:99
-
 export JOB=kibana-osquery-cypress
 
 echo "--- Osquery Cypress tests"

.buildkite/scripts/steps/functional/response_ops.sh

@@ -3,14 +3,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-Xvfb -screen 0 1680x946x24 :99 &
-
-export DISPLAY=:99
-
 echo "--- Response Ops Cypress Tests on Security Solution"
 
 yarn --cwd x-pack/plugins/security_solution cypress:run:respops

.buildkite/scripts/steps/functional/response_ops_cases.sh

@@ -3,14 +3,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-Xvfb -screen 0 1680x946x24 :99 &
-
-export DISPLAY=:99
-
 echo "--- Response Ops Cases Cypress Tests on Security Solution"
 
 yarn --cwd x-pack/plugins/security_solution cypress:run:cases

.buildkite/scripts/steps/functional/security_solution.sh

@@ -3,14 +3,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-Xvfb :99 -screen 0 1600x1200x24 &
-
-export DISPLAY=:99
-
 echo "--- Security Solution tests (Chrome)"
 
 yarn --cwd x-pack/plugins/security_solution cypress:run

.buildkite/scripts/steps/functional/security_solution_explore.sh

@@ -3,15 +3,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-
-Xvfb :99 -screen 0 1600x1200x24 &
-
-export DISPLAY=:99
-
 echo "--- Explore Cypress Tests on Security Solution"
 
 yarn --cwd x-pack/plugins/security_solution cypress:explore:run

.buildkite/scripts/steps/functional/security_solution_investigations.sh

@@ -3,15 +3,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-
-Xvfb :99 -screen 0 1600x1200x24 &
-
-export DISPLAY=:99
-
 echo "--- Investigations Cypress Tests on Security Solution"
 
 yarn --cwd x-pack/plugins/security_solution cypress:investigations:run

.buildkite/scripts/steps/functional/serverless_ftr.sh

@@ -15,11 +15,13 @@ elif [[ "$SERVERLESS_ENVIRONMENT" == "observability" ]]; then
   SERVERLESS_CONFIGS=(
     "x-pack/test_serverless/api_integration/test_suites/observability/config.ts"
     "x-pack/test_serverless/functional/test_suites/observability/config.ts"
+    "x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts"
   )
 elif [[ "$SERVERLESS_ENVIRONMENT" == "security" ]]; then
   SERVERLESS_CONFIGS=(
     "x-pack/test_serverless/api_integration/test_suites/security/config.ts"
     "x-pack/test_serverless/functional/test_suites/security/config.ts"
+    "x-pack/test_serverless/functional/test_suites/security/cypress/config_headless.ts"
   )
 fi
 

.buildkite/scripts/steps/functional/threat_intelligence.sh

@@ -3,14 +3,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-threat-intelligence-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
-Xvfb :99 -screen 0 1600x1200x24 &
-
-export DISPLAY=:99
-
 echo "--- Threat Intelligence tests (Chrome)"
 
 yarn --cwd x-pack/plugins/threat_intelligence cypress:run

.eslintrc.js

@@ -986,7 +986,11 @@ module.exports = {
         'x-pack/packages/kbn-elastic-assistant/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/security-solution/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/security_solution/public/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/security_solution_ess/public/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/security_solution_serverless/public/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/security_solution/common/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/security_solution_ess/common/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/security_solution_serverless/common/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/timelines/public/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/timelines/common/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/cases/public/**/*.{js,mjs,ts,tsx}',
@@ -1014,6 +1018,8 @@ module.exports = {
         'x-pack/packages/kbn-elastic-assistant/**/*.{ts,tsx}',
         'x-pack/packages/security-solution/**/*.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{ts,tsx}',
+        'x-pack/plugins/security_solution_ess/**/*.{ts,tsx}',
+        'x-pack/plugins/security_solution_serverless/**/*.{ts,tsx}',
         'x-pack/plugins/timelines/**/*.{ts,tsx}',
         'x-pack/plugins/cases/**/*.{ts,tsx}',
       ],
@@ -1022,6 +1028,8 @@ module.exports = {
         'x-pack/packages/kbn-elastic-assistant/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/packages/security-solution/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{test,mock,test_helper}.{ts,tsx}',
+        'x-pack/plugins/security_solution_ess/**/*.{test,mock,test_helper}.{ts,tsx}',
+        'x-pack/plugins/security_solution_serverless/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/plugins/timelines/**/*.{test,mock,test_helper}.{ts,tsx}',
         'x-pack/plugins/cases/**/*.{test,mock,test_helper}.{ts,tsx}',
       ],
@@ -1036,6 +1044,8 @@ module.exports = {
         'x-pack/packages/kbn-elastic-assistant/**/*.{ts,tsx}',
         'x-pack/packages/security-solution/**/*.{ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{ts,tsx}',
+        'x-pack/plugins/security_solution_ess/**/*.{ts,tsx}',
+        'x-pack/plugins/security_solution_serverless/**/*.{ts,tsx}',
         'x-pack/plugins/timelines/**/*.{ts,tsx}',
         'x-pack/plugins/cases/**/*.{ts,tsx}',
       ],
@@ -1069,6 +1079,8 @@ module.exports = {
         'x-pack/packages/kbn-elastic-assistant/**/*.{js,mjs,ts,tsx}',
         'x-pack/packages/security-solution/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/security_solution/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/security_solution_ess/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/timelines/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
       ],
@@ -1163,6 +1175,8 @@ module.exports = {
         {
           files: [
             'x-pack/plugins/security_solution/**/*.{js,mjs,ts,tsx}',
+            'x-pack/plugins/security_solution_ess/**/*.{js,mjs,ts,tsx}',
+            'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
             'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
           ],
           rules: {