Merge branch 'main' into obs-ai-assistant-ccs-by-default

.buildkite/ftr_platform_stateful_configs.yml

@@ -34,7 +34,9 @@ disabled:
 
   # Cypress configs, for now these are still run manually
   - x-pack/test/fleet_cypress/cli_config.ts
+  - x-pack/test/fleet_cypress/cli_config.space_awareness.ts
   - x-pack/test/fleet_cypress/config.ts
+  - x-pack/test/fleet_cypress/config.space_awareness.ts
   - x-pack/test/fleet_cypress/visual_config.ts
 
 defaultQueue: 'n2-4-spot'
@@ -315,6 +317,7 @@ enabled:
   - x-pack/test/security_api_integration/saml.http2.config.ts
   - x-pack/test/security_api_integration/saml_cloud.config.ts
   - x-pack/test/security_api_integration/chips.config.ts
+  - x-pack/test/security_api_integration/features.config.ts
   - x-pack/test/security_api_integration/session_idle.config.ts
   - x-pack/test/security_api_integration/session_invalidate.config.ts
   - x-pack/test/security_api_integration/session_lifespan.config.ts

.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml

@@ -44,3 +44,4 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
       tags:
         - kibana
+        - kibana-serverless-release

.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml

@@ -0,0 +1,44 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-deploy-project-from-pr
+  description: 'Builds and deploys a Kibana serverless project from a PR'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-deploy-project-from-pr'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: 'group:kibana-operations'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / deploy project from PR
+      description: 'Builds and deploys a Kibana serverless project from a PR'
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
+
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml
+      skip_intermediate_builds: true
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      tags:
+        - kibana

.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml

@@ -55,3 +55,4 @@ spec:
           branch: main
       tags:
         - kibana
+        - kibana-serverless-release

.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

@@ -46,19 +46,19 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
       schedules:
         Daily build (main):
-          cronline: 0 9 * * * America/New_York
+          cronline: 0 22 * * * America/New_York
           message: Daily build
           branch: main
         Daily build (8.x):
-          cronline: 0 9 * * * America/New_York
+          cronline: 0 22 * * * America/New_York
           message: Daily build
           branch: '8.x'
         Daily build (8.15):
-          cronline: 0 9 * * * America/New_York
+          cronline: 0 22 * * * America/New_York
           message: Daily build
           branch: '8.15'
         Daily build (7.17):
-          cronline: 0 9 * * * America/New_York
+          cronline: 0 22 * * * America/New_York
           message: Daily build
           branch: '7.17'
       tags:

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -49,3 +49,4 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
       tags:
         - kibana
+        - kibana-serverless-release

.buildkite/pipeline-resource-definitions/kibana-pr.yml

@@ -20,8 +20,8 @@ spec:
     spec:
       env:
         ELASTIC_PR_COMMENTS_ENABLED: 'true'
-        ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
-        ELASTIC_GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
         GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci
       allow_rebuilds: true
       branch_configuration: ''

.buildkite/pipeline-resource-definitions/kibana-serverless-emergency-release.yml

@@ -30,3 +30,4 @@ spec:
           access_level: READ_ONLY
       tags:
         - kibana
+        - kibana-serverless-release

.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml

@@ -33,3 +33,4 @@ spec:
           access_level: READ_ONLY
       tags:
         - kibana
+        - kibana-serverless-release

.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml

@@ -33,3 +33,4 @@ spec:
           access_level: READ_ONLY
       tags:
         - kibana
+        - kibana-serverless-release

.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml

@@ -46,3 +46,4 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
       tags:
         - kibana
+        - kibana-serverless-release

.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml

@@ -48,3 +48,4 @@ spec:
           branch: main
       tags:
         - kibana
+        - kibana-serverless-release

.buildkite/pipeline-resource-definitions/kibana-vm-images.yml

@@ -0,0 +1,48 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-vm-images
+  description: Build CI agent VM images for Kibana
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-vm-images'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / vm images
+      description: Build CI agent VM images for Kibana
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      default_branch: main
+      repository: elastic/ci-agent-images
+      pipeline_file: vm-images/.buildkite/pipeline.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        trigger_mode: none
+      schedules:
+        daily kibana image build:
+          branch: main
+          cronline: '0 0 * * *'
+          env:
+            IMAGES_CONFIG: kibana/images.yml
+          message: Builds Kibana VM images daily
+        daily kibana fips image build:
+          branch: main
+          cronline: '0 4 * * *' # make sure this runs after the daily kibana image build
+          env:
+            BASE_IMAGES_CONFIG: 'core/images.yml,kibana/images.yml'
+            IMAGES_CONFIG: kibana/fips.yml
+          message: Builds Kibana FIPS VM image daily
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -16,6 +16,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-chrome-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
@@ -36,6 +37,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-vm-images.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/scalability_testing-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -54,7 +54,7 @@ steps:
         env:
           FTR_CONFIGS_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh"
           JEST_INTEGRATION_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/jest_integration.sh"
-          FTR_CONFIG_PATTERNS: "**/test_serverless/**,**/test/security_solution_api_integration/**/serverless.config.ts"
+          FTR_CONFIG_PATTERNS: "**/test_serverless/**,**/test/security_solution_api_integration/**/serverless.config.ts,x-pack/test/api_integration/deployment_agnostic/configs/serverless/**"
           FTR_EXTRA_ARGS: "$FTR_EXTRA_ARGS"
           LIMIT_CONFIG_TYPE: "functional,integration"
         retry:

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml

@@ -10,7 +10,7 @@ steps:
       env:
         TARGET_ENV: production
         CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
+        CHECK_SLO_TAG: kbn-quality-gate
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
         DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-staging.yaml

@@ -46,3 +46,4 @@ steps:
         command: "make -C /agent trigger-manual-verification-phase"
         agents:
           image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
+        if: build.env("DEPLOYMENT_SLICES") =~ /.*staging-ds-2.*/

.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml

@@ -10,7 +10,7 @@ steps:
       env:
         TARGET_ENV: production
         CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
+        CHECK_SLO_TAG: kbn-quality-gate
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
         DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -46,3 +46,4 @@ steps:
         command: "make -C /agent trigger-manual-verification-phase"
         agents:
           image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
+        if: build.env("DEPLOYMENT_SLICES") =~ /.*staging-ds-2.*/

.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml

@@ -0,0 +1,53 @@
+agents:
+  provider: gcp
+  image: family/kibana-ubuntu-2004
+  imageProject: elastic-images-prod
+
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      machineType: n2-standard-16
+      preemptible: true
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 90
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/artifacts/docker_image.sh
+    label: 'Build Project Image'
+    key: build_project_image
+    agents:
+      machineType: n2-standard-16
+      preemptible: true
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/serverless/deploy.sh
+    label: 'Deploy Project'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    timeout_in_minutes: 10

.buildkite/pull_requests.json

@@ -30,7 +30,8 @@
         "^\\.backportrc\\.json$",
         "^nav-kibana-dev\\.docnav\\.json$",
         "^src/dev/prs/kibana_qa_pr_list\\.json$",
-        "^\\.buildkite/pull_requests\\.json$"
+        "^\\.buildkite/pull_requests\\.json$",
+        "^\\.devcontainer/"
       ],
       "always_require_ci_on_changed": [
         "^docs/developer/plugin-list.asciidoc$",
@@ -46,6 +47,30 @@
         "/__snapshots__/",
         "\\.test\\.(ts|tsx|js|jsx)"
       ]
+    },
+    {
+      "repoOwner": "elastic",
+      "repoName": "kibana",
+      "pipelineSlug": "kibana-deploy-project-from-pr",
+
+      "enabled": true,
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": false,
+      "build_drafts": false,
+      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:project))$",
+      "kibana_versions_check": true,
+      "kibana_build_reuse": true,
+      "kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge", "kibana-deploy-project-from-pr"],
+      "kibana_build_reuse_regexes": [
+        "^test/",
+        "^x-pack/test/",
+        "/__snapshots__/",
+        "\\.test\\.(ts|tsx|js|jsx)"
+      ]
     }
   ]
 }

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -20,8 +20,8 @@ if (!prConfig) {
 }
 
 const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
-const REQUIRED_PATHS = prConfig.always_require_ci_on_changed.map((r) => new RegExp(r, 'i'));
-const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed.map((r) => new RegExp(r, 'i'));
+const REQUIRED_PATHS = prConfig.always_require_ci_on_changed!.map((r) => new RegExp(r, 'i'));
+const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed!.map((r) => new RegExp(r, 'i'));
 
 const getPipeline = (filename: string, removeSteps = true) => {
   const str = fs.readFileSync(filename).toString();

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -51,7 +51,7 @@ fi
 if is_pr_with_label "ci:cloud-redeploy"; then
   echo "--- Shutdown Previous Deployment"
   CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
-  if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
+  if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" == "null" ]; then
     echo "No deployment to remove"
   else
     echo "Shutting down previous deployment..."

.buildkite/scripts/steps/es_snapshots/promote.sh

@@ -16,4 +16,12 @@ ts-node "$(dirname "${0}")/promote_manifest.ts" "$ES_SNAPSHOT_MANIFEST"
 if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
   echo "--- Trigger agent packer cache pipeline"
   ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-agent-packer-cache main
+  cat << EOF | buildkite-agent pipeline upload
+steps:
+  - label: "Builds Kibana VM images for cache update"
+    trigger: ci-vm-images
+    build:
+      env:
+        IMAGES_CONFIG="kibana/images.yml"
+EOF
 fi

.buildkite/scripts/steps/functional/fleet_cypress.sh

@@ -12,4 +12,4 @@ echo "--- Fleet Cypress tests (Chrome)"
 cd x-pack/plugins/fleet
 
 set +e
-yarn cypress:run:reporter; status=$?; yarn junit:merge || :; exit $status
+yarn cypress:run:reporter; status=$?; yarn cypress_space_awareness:run:reporter; space_status=$?; yarn junit:merge || :; [ "$status" -ne 0 ] && exit $status || [ "$space_status" -ne 0 ] && exit $space_status || exit 0

.buildkite/scripts/steps/serverless/deploy.sh

@@ -56,7 +56,7 @@ deploy() {
 
   PROJECT_ID=$(jq -r '[.items[] | select(.name == "'$PROJECT_NAME'")] | .[0].id' $PROJECT_EXISTS_LOGS)
   if is_pr_with_label "ci:project-redeploy"; then
-    if [ -z "${PROJECT_ID}" ]; then
+    if [ -z "${PROJECT_ID}" ] || [ "${PROJECT_ID}" == "null" ]; then
       echo "No project to remove"
     else
       echo "Shutting down previous project..."
@@ -159,13 +159,13 @@ EOF
 }
 
 is_pr_with_label "ci:project-deploy-elasticsearch" && deploy "elasticsearch"
+is_pr_with_label "ci:project-deploy-security" && deploy "security"
 if is_pr_with_label "ci:project-deploy-observability" ; then
   # Only deploy observability if the PR is targeting main
   if [[ "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" == "main" ]]; then
     create_github_issue_oblt_test_environments
     buildkite-agent annotate --context obl-test-info --style info 'See linked [Deploy Serverless Kibana] issue in pull request for project deployment information'
   fi
 fi
-is_pr_with_label "ci:project-deploy-security" && deploy "security"
 
 exit 0;