Merge branch 'main' into perf/add-journeys-with-kibana-http2

.buildkite/ftr_oblt_serverless_configs.yml

@@ -6,10 +6,6 @@ disabled:
   - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts
   - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts
 
-    # serverless config files that run deployment-agnostic tests
-    # Failing https://github.com/elastic/kibana/issues/195811
-  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
-
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/observability/config.ts
@@ -29,3 +25,5 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -320,6 +320,7 @@ enabled:
   - x-pack/test/security_api_integration/saml.http2.config.ts
   - x-pack/test/security_api_integration/saml_cloud.config.ts
   - x-pack/test/security_api_integration/chips.config.ts
+  - x-pack/test/security_api_integration/features.config.ts
   - x-pack/test/security_api_integration/session_idle.config.ts
   - x-pack/test/security_api_integration/session_invalidate.config.ts
   - x-pack/test/security_api_integration/session_lifespan.config.ts

.buildkite/ftr_search_serverless_configs.yml

@@ -1,10 +1,6 @@
 disabled:
   # Base config files, only necessary to inform config finding script
 
-  # serverless config files that run deployment-agnostic tests
-  # Failing https://github.com/elastic/kibana/issues/195811
-  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts
-
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/search/config.ts
@@ -22,3 +18,5 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -20,10 +20,6 @@ disabled:
   - x-pack/test_serverless/functional/config.base.ts
   - x-pack/test_serverless/shared/config.base.ts
 
-  # serverless config files that run deployment-agnostic tests
-  # Failing https://github.com/elastic/kibana/issues/195811
-  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts
-
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/security/config.ts
@@ -104,3 +100,5 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts

.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml

@@ -0,0 +1,44 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-deploy-project-from-pr
+  description: 'Builds and deploys a Kibana serverless project from a PR'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-deploy-project-from-pr'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: 'group:kibana-operations'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / deploy project from PR
+      description: 'Builds and deploys a Kibana serverless project from a PR'
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
+
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml
+      skip_intermediate_builds: true
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      tags:
+        - kibana

.buildkite/pipeline-resource-definitions/kibana-pr.yml

@@ -20,8 +20,8 @@ spec:
     spec:
       env:
         ELASTIC_PR_COMMENTS_ENABLED: 'true'
-        ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
-        ELASTIC_GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
         GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci
       allow_rebuilds: true
       branch_configuration: ''

.buildkite/pipeline-resource-definitions/kibana-vm-images.yml

@@ -0,0 +1,48 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-vm-images
+  description: Build CI agent VM images for Kibana
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-vm-images'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / vm images
+      description: Build CI agent VM images for Kibana
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      default_branch: main
+      repository: elastic/ci-agent-images
+      pipeline_file: vm-images/.buildkite/pipeline.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        trigger_mode: none
+      schedules:
+        daily kibana image build:
+          branch: main
+          cronline: '0 0 * * *'
+          env:
+            IMAGES_CONFIG: kibana/images.yml
+          message: Builds Kibana VM images daily
+        daily kibana fips image build:
+          branch: main
+          cronline: '0 4 * * *' # make sure this runs after the daily kibana image build
+          env:
+            BASE_IMAGES_CONFIG: 'core/images.yml,kibana/images.yml'
+            IMAGES_CONFIG: kibana/fips.yml
+          message: Builds Kibana FIPS VM image daily
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -16,6 +16,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-chrome-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
@@ -36,6 +37,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-vm-images.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/scalability_testing-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -54,7 +54,7 @@ steps:
         env:
           FTR_CONFIGS_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh"
           JEST_INTEGRATION_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/jest_integration.sh"
-          FTR_CONFIG_PATTERNS: "**/test_serverless/**,**/test/security_solution_api_integration/**/serverless.config.ts"
+          FTR_CONFIG_PATTERNS: "**/test_serverless/**,**/test/security_solution_api_integration/**/serverless.config.ts,x-pack/test/api_integration/deployment_agnostic/configs/serverless/**"
           FTR_EXTRA_ARGS: "$FTR_EXTRA_ARGS"
           LIMIT_CONFIG_TYPE: "functional,integration"
         retry:

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml

@@ -10,7 +10,7 @@ steps:
       env:
         TARGET_ENV: production
         CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
+        CHECK_SLO_TAG: kbn-quality-gate
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
         DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-staging.yaml

@@ -46,3 +46,4 @@ steps:
         command: "make -C /agent trigger-manual-verification-phase"
         agents:
           image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
+        if: build.env("DEPLOYMENT_SLICES") =~ /.*staging-ds-2.*/

.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml

@@ -10,7 +10,7 @@ steps:
       env:
         TARGET_ENV: production
         CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
+        CHECK_SLO_TAG: kbn-quality-gate
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
         DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -46,3 +46,4 @@ steps:
         command: "make -C /agent trigger-manual-verification-phase"
         agents:
           image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
+        if: build.env("DEPLOYMENT_SLICES") =~ /.*staging-ds-2.*/

.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml

@@ -0,0 +1,53 @@
+agents:
+  provider: gcp
+  image: family/kibana-ubuntu-2004
+  imageProject: elastic-images-prod
+
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      machineType: n2-standard-16
+      preemptible: true
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 90
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/artifacts/docker_image.sh
+    label: 'Build Project Image'
+    key: build_project_image
+    agents:
+      machineType: n2-standard-16
+      preemptible: true
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/serverless/deploy.sh
+    label: 'Deploy Project'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    timeout_in_minutes: 10

.buildkite/pull_requests.json

@@ -47,6 +47,30 @@
         "/__snapshots__/",
         "\\.test\\.(ts|tsx|js|jsx)"
       ]
+    },
+    {
+      "repoOwner": "elastic",
+      "repoName": "kibana",
+      "pipelineSlug": "kibana-deploy-project-from-pr",
+
+      "enabled": true,
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": false,
+      "build_drafts": false,
+      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:project))$",
+      "kibana_versions_check": true,
+      "kibana_build_reuse": true,
+      "kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge", "kibana-deploy-project-from-pr"],
+      "kibana_build_reuse_regexes": [
+        "^test/",
+        "^x-pack/test/",
+        "/__snapshots__/",
+        "\\.test\\.(ts|tsx|js|jsx)"
+      ]
     }
   ]
 }

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -20,8 +20,8 @@ if (!prConfig) {
 }
 
 const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
-const REQUIRED_PATHS = prConfig.always_require_ci_on_changed.map((r) => new RegExp(r, 'i'));
-const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed.map((r) => new RegExp(r, 'i'));
+const REQUIRED_PATHS = prConfig.always_require_ci_on_changed!.map((r) => new RegExp(r, 'i'));
+const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed!.map((r) => new RegExp(r, 'i'));
 
 const getPipeline = (filename: string, removeSteps = true) => {
   const str = fs.readFileSync(filename).toString();

.buildkite/scripts/steps/serverless/deploy.sh

@@ -159,13 +159,13 @@ EOF
 }
 
 is_pr_with_label "ci:project-deploy-elasticsearch" && deploy "elasticsearch"
+is_pr_with_label "ci:project-deploy-security" && deploy "security"
 if is_pr_with_label "ci:project-deploy-observability" ; then
   # Only deploy observability if the PR is targeting main
   if [[ "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" == "main" ]]; then
     create_github_issue_oblt_test_environments
     buildkite-agent annotate --context obl-test-info --style info 'See linked [Deploy Serverless Kibana] issue in pull request for project deployment information'
   fi
 fi
-is_pr_with_label "ci:project-deploy-security" && deploy "security"
 
 exit 0;

.buildkite/scripts/steps/storybooks/build_and_upload.ts

@@ -43,6 +43,7 @@ const STORYBOOKS = [
   'lists',
   'observability',
   'observability_ai_assistant',
+  'observability_shared',
   'presentation',
   'security_solution',
   'security_solution_packages',