Merge branch 'main' into embeddableRebuild/referenceExtraction

.buildkite/ftr_configs.yml

@@ -521,6 +521,8 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/genai/invoke_ai/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/ess.config.ts

.buildkite/pipelines/emergency_release.yml

@@ -1,3 +1,18 @@
+env:
+  VERSION: ${BUILDKITE_COMMIT:0:12}
+  EMERGENCY_RELEASE_REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/emergency.yaml
+
 steps:
-  - label: Placeholder
-    command: echo "Hello!"
+  - label: ':label: Create tag'
+    command: .buildkite/scripts/serverless/create_deploy_tag/create_fix_tag.sh
+    if: build.branch =~ /^deploy-fix@.*\$/
+
+  - wait
+
+  - label: ':argo: Emergency Release'
+    if: build.branch =~ /^deploy-fix@.*\$/
+    trigger: gpctl-promote
+    build:
+      env:
+        REMOTE_SERVICE_CONFIG: "${EMERGENCY_RELEASE_REMOTE_SERVICE_CONFIG}"
+        SERVICE_COMMIT_HASH: "${VERSION}"

.buildkite/pipelines/on_merge.yml

@@ -372,7 +372,7 @@ steps:
     agents:
       queue: n2-8-spot
     key: storybooks
-    timeout_in_minutes: 60
+    timeout_in_minutes: 80
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/base.yml

@@ -96,7 +96,7 @@ steps:
     agents:
       queue: n2-4-spot
     key: build_api_docs
-    timeout_in_minutes: 60
+    timeout_in_minutes: 90
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/fips.yml

@@ -0,0 +1,14 @@
+steps:
+  - command: .buildkite/scripts/steps/fips/build.sh
+    label: 'Build FIPS Image'
+    agents:
+      queue: n2-2-spot
+    depends_on:
+      - build
+      - quick_checks
+    timeout_in_minutes: 60
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/pull_request/storybooks.yml

@@ -4,4 +4,4 @@ steps:
     agents:
       queue: n2-8-spot
     key: storybooks
-    timeout_in_minutes: 60
+    timeout_in_minutes: 80

.buildkite/pipelines/security_solution/api_integration.yml

@@ -146,6 +146,17 @@ steps:
             - exit_status: "1"
               limit: 2
 
+      - label: Running genai:qa:serverless
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh genai:qa:serverless
+        key: genai:qa:serverless
+        agents:
+          queue: n2-4-spot
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: "1"
+              limit: 2
+
       - label: Running prebuilt_rules_management:qa:serverless
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh prebuilt_rules_management:qa:serverless
         key: prebuilt_rules_management:qa:serverless

.buildkite/scripts/build_kibana.sh

@@ -31,6 +31,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --docker-tag-qualifier="$GIT_COMMIT" \
   --docker-push \
   --skip-docker-ubi \
+  --skip-docker-fips \
   --skip-docker-ubuntu \
   --skip-docker-serverless \
   --skip-docker-contexts

.buildkite/scripts/lifecycle/post_build.sh

@@ -12,7 +12,7 @@ fi
 ts-node "$(dirname "${0}")/ci_stats_complete.ts"
 
 if [[ "${GITHUB_PR_NUMBER:-}" ]]; then
-  DOCS_CHANGES_URL="https://kibana_$GITHUB_PR_NUMBER}.docs-preview.app.elstc.co/diff"
+  DOCS_CHANGES_URL="https://kibana_bk_$GITHUB_PR_NUMBER}.docs-preview.app.elstc.co/diff"
   DOCS_CHANGES=$(curl --connect-timeout 10 -m 10 -sf "$DOCS_CHANGES_URL" || echo '')
 
   if [[ "$DOCS_CHANGES" && "$DOCS_CHANGES" != "There aren't any differences!" ]]; then

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -139,6 +139,10 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
     }
 
+    if (GITHUB_PR_LABELS.includes('ci:build-docker-fips')) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/fips.yml'));
+    }
+
     if (
       GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
       GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||

.buildkite/scripts/serverless/create_deploy_tag/create_fix_tag.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "--- Verify $BUILDKITE_COMMIT exists in $BUILDKITE_BRANCH"
+# Step 1: Check if the commit is in the specific named branch
+if git merge-base --is-ancestor $BUILDKITE_COMMIT $BUILDKITE_BRANCH; then
+    echo "Commit $BUILDKITE_COMMIT is part of the $BUILDKITE_BRANCH branch"
+
+    # Step 2: Check if the commit is also part of any other branches
+    # This command lists all branches containing the commit and counts them
+    branches_containing_commit=$(git branch --contains $BUILDKITE_COMMIT | wc -l)
+
+    # If the commit is in more than one branch, exit with non-zero
+    if [[ $branches_containing_commit -gt 1 ]]; then
+        echo "Error: Commit $BUILDKITE_COMMIT is part of multiple branches."
+        exit 1
+    else
+        echo "Commit $BUILDKITE_COMMIT is uniquely part of the $BUILDKITE_BRANCH branch."
+    fi
+else
+    echo "Commit $BUILDKITE_COMMIT is not part of the $BUILDKITE_BRANCH branch."
+    exit 1
+fi
+
+echo "--- Create tag $BUILDKITE_BRANCH"
+git tag "$BUILDKITE_BRANCH" "$BUILDKITE_COMMIT"
+
+echo "--- Push tag $BUILDKITE_BRANCH"
+git push origin tag "$BUILDKITE_BRANCH"

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -36,6 +36,7 @@ node scripts/build \
   --docker-tag="$KIBANA_IMAGE_TAG" \
   --skip-docker-ubuntu \
   --skip-docker-ubi \
+  --skip-docker-fips \
   --skip-docker-cloud \
   --skip-docker-contexts
 

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -42,6 +42,7 @@ else
     --docker-tag-qualifier="$GIT_COMMIT" \
     --docker-push \
     --skip-docker-ubi \
+    --skip-docker-fips \
     --skip-docker-ubuntu \
     --skip-docker-serverless \
     --skip-docker-contexts

.buildkite/scripts/steps/demo_env/kibana.sh

@@ -9,7 +9,7 @@ source "$(dirname "${0}")/config.sh"
 export KIBANA_IMAGE="gcr.io/elastic-kibana-184716/demo/kibana:$DEPLOYMENT_NAME-$(git rev-parse HEAD)"
 
 echo '--- Build Kibana'
-node scripts/build --debug --docker-images --example-plugins --skip-docker-ubi --skip-docker-cloud --skip-docker-serverless --skip-docker-contexts
+node scripts/build --debug --docker-images --example-plugins --skip-docker-ubi --skip-docker-fips --skip-docker-cloud --skip-docker-serverless --skip-docker-contexts
 
 echo '--- Build Docker image with example plugins'
 cd target/example_plugins

.buildkite/scripts/steps/fips/build.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/artifacts/env.sh
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+mkdir -p target
+download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+
+echo "--- Build FIPS image"
+node scripts/build \
+    --skip-initialize \
+    --skip-generic-folders \
+    --skip-platform-folders \
+    --skip-cdn-assets \
+    --skip-archives \
+    --docker-images \
+    --docker-namespace="kibana-ci" \
+    --docker-tag-qualifier="$BUILDKITE_COMMIT" \
+    --docker-push \
+    --skip-docker-ubi \
+    --skip-docker-ubuntu \
+    --skip-docker-cloud \
+    --skip-docker-serverless \
+    --skip-docker-contexts
+
+docker logout docker.elastic.co
+
+# Moving to `target/` first will keep `buildkite-agent` from including directories in the artifact name
+cd "$KIBANA_DIR/target"
+buildkite-agent artifact upload "./*docker-image*.tar.gz"
+
+KIBANA_UBI_FIPS_IMAGE="docker.elastic.co/kibana-ci/kibana-ubi-fips:$FULL_VERSION-$BUILDKITE_COMMIT"
+
+cat <<EOF | buildkite-agent annotate --style "info" --context fips
+  ### Kibana FIPS Image
+
+  UBI image: \`$KIBANA_UBI_FIPS_IMAGE\`
+EOF
+
+buildkite-agent meta-data set pr_comment:build_fips:head "* Kibana UBI FIPS Image: \`$KIBANA_UBI_FIPS_IMAGE\`"
+buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"

.buildkite/scripts/steps/package_testing/build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-contexts --skip-cdn-assets
+node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-serverless --skip-docker-ubi --skip-docker-fips --skip-docker-contexts --skip-cdn-assets
 
 DOCKER_FILE="kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz"
 

.github/CODEOWNERS

@@ -324,6 +324,7 @@ src/plugins/data_view_editor @elastic/kibana-data-discovery
 examples/data_view_field_editor_example @elastic/kibana-data-discovery
 src/plugins/data_view_field_editor @elastic/kibana-data-discovery
 src/plugins/data_view_management @elastic/kibana-data-discovery
+packages/kbn-data-view-utils @elastic/kibana-data-discovery
 src/plugins/data_views @elastic/kibana-data-discovery
 x-pack/plugins/data_visualizer @elastic/ml-ui
 x-pack/plugins/dataset_quality @elastic/obs-ux-logs-team
@@ -673,6 +674,7 @@ packages/kbn-search-index-documents @elastic/enterprise-search-frontend
 packages/kbn-search-response-warnings @elastic/kibana-data-discovery
 x-pack/plugins/searchprofiler @elastic/platform-deployment-management
 x-pack/test/security_api_integration/packages/helpers @elastic/kibana-security
+packages/kbn-security-hardening @elastic/kibana-security
 x-pack/plugins/security @elastic/kibana-security
 x-pack/packages/security/plugin_types_common @elastic/kibana-security
 x-pack/packages/security/plugin_types_public @elastic/kibana-security
@@ -1500,6 +1502,9 @@ x-pack/plugins/security_solution/public/flyout/entity_details @elastic/security-
 x-pack/plugins/security_solution/common/api/entity_analytics @elastic/security-entity-analytics
 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/risk_score @elastic/security-entity-analytics
 
+## Security Solution sub teams - GenAI
+x-pack/test/security_solution_api_integration/test_suites/genai @elastic/security-generative-ai
+
 # Security Defend Workflows - OSQuery Ownership
 /x-pack/plugins/security_solution/common/api/detection_engine/model/rule_response_actions @elastic/security-defend-workflows
 /x-pack/plugins/security_solution/public/detection_engine/rule_response_actions @elastic/security-defend-workflows

api_docs/actions.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/actions
 title: "actions"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the actions plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
 ---
 import actionsObj from './actions.devdocs.json';

api_docs/advanced_settings.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/advancedSettings
 title: "advancedSettings"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the advancedSettings plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
 ---
 import advancedSettingsObj from './advanced_settings.devdocs.json';

api_docs/ai_assistant_management_observability.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiAssistantManagementObservability
 title: "aiAssistantManagementObservability"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiAssistantManagementObservability plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiAssistantManagementObservability']
 ---
 import aiAssistantManagementObservabilityObj from './ai_assistant_management_observability.devdocs.json';

api_docs/ai_assistant_management_selection.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiAssistantManagementSelection
 title: "aiAssistantManagementSelection"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiAssistantManagementSelection plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiAssistantManagementSelection']
 ---
 import aiAssistantManagementSelectionObj from './ai_assistant_management_selection.devdocs.json';

api_docs/aiops.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiops
 title: "aiops"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiops plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops']
 ---
 import aiopsObj from './aiops.devdocs.json';

api_docs/alerting.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/alerting
 title: "alerting"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the alerting plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'alerting']
 ---
 import alertingObj from './alerting.devdocs.json';

api_docs/apm.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apm
 title: "apm"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apm plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apm']
 ---
 import apmObj from './apm.devdocs.json';

api_docs/apm_data_access.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apmDataAccess
 title: "apmDataAccess"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apmDataAccess plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apmDataAccess']
 ---
 import apmDataAccessObj from './apm_data_access.devdocs.json';

api_docs/asset_manager.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/assetManager
 title: "assetManager"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the assetManager plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'assetManager']
 ---
 import assetManagerObj from './asset_manager.devdocs.json';

api_docs/banners.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/banners
 title: "banners"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the banners plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'banners']
 ---
 import bannersObj from './banners.devdocs.json';

api_docs/bfetch.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/bfetch
 title: "bfetch"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the bfetch plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'bfetch']
 ---
 import bfetchObj from './bfetch.devdocs.json';

api_docs/canvas.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/canvas
 title: "canvas"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the canvas plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'canvas']
 ---
 import canvasObj from './canvas.devdocs.json';

api_docs/cases.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/cases
 title: "cases"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the cases plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'cases']
 ---
 import casesObj from './cases.devdocs.json';

api_docs/charts.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/charts
 title: "charts"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the charts plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'charts']
 ---
 import chartsObj from './charts.devdocs.json';

api_docs/cloud.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/cloud
 title: "cloud"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the cloud plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'cloud']
 ---
 import cloudObj from './cloud.devdocs.json';

api_docs/cloud_data_migration.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/cloudDataMigration
 title: "cloudDataMigration"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the cloudDataMigration plugin
-date: 2024-02-07
+date: 2024-02-09
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'cloudDataMigration']
 ---
 import cloudDataMigrationObj from './cloud_data_migration.devdocs.json';