Merge branch 'main' into 2024_09_10-fix_long_user_messages

.buildkite/ftr_platform_stateful_configs.yml

@@ -34,7 +34,9 @@ disabled:
 
   # Cypress configs, for now these are still run manually
   - x-pack/test/fleet_cypress/cli_config.ts
+  - x-pack/test/fleet_cypress/cli_config.space_awareness.ts
   - x-pack/test/fleet_cypress/config.ts
+  - x-pack/test/fleet_cypress/config.space_awareness.ts
   - x-pack/test/fleet_cypress/visual_config.ts
 
 defaultQueue: 'n2-4-spot'

.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml

@@ -0,0 +1,44 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-deploy-project-from-pr
+  description: 'Builds and deploys a Kibana serverless project from a PR'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-deploy-project-from-pr'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: 'group:kibana-operations'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / deploy project from PR
+      description: 'Builds and deploys a Kibana serverless project from a PR'
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
+
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml
+      skip_intermediate_builds: true
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      tags:
+        - kibana

.buildkite/pipeline-resource-definitions/kibana-pr.yml

@@ -20,8 +20,8 @@ spec:
     spec:
       env:
         ELASTIC_PR_COMMENTS_ENABLED: 'true'
-        ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
-        ELASTIC_GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+        GITHUB_STEP_COMMIT_STATUS_ENABLED: 'true'
         GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci
       allow_rebuilds: true
       branch_configuration: ''

.buildkite/pipeline-resource-definitions/locations.yml

@@ -16,6 +16,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-chrome-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -54,7 +54,7 @@ steps:
         env:
           FTR_CONFIGS_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh"
           JEST_INTEGRATION_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/jest_integration.sh"
-          FTR_CONFIG_PATTERNS: "**/test_serverless/**,**/test/security_solution_api_integration/**/serverless.config.ts"
+          FTR_CONFIG_PATTERNS: "**/test_serverless/**,**/test/security_solution_api_integration/**/serverless.config.ts,x-pack/test/api_integration/deployment_agnostic/configs/serverless/**"
           FTR_EXTRA_ARGS: "$FTR_EXTRA_ARGS"
           LIMIT_CONFIG_TYPE: "functional,integration"
         retry:

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml

@@ -10,7 +10,7 @@ steps:
       env:
         TARGET_ENV: production
         CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
+        CHECK_SLO_TAG: kbn-quality-gate
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
         DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-staging.yaml

@@ -46,3 +46,4 @@ steps:
         command: "make -C /agent trigger-manual-verification-phase"
         agents:
           image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
+        if: build.env("DEPLOYMENT_SLICES") =~ /.*staging-ds-2.*/

.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml

@@ -10,7 +10,7 @@ steps:
       env:
         TARGET_ENV: production
         CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
+        CHECK_SLO_TAG: kbn-quality-gate
         CHECK_SLO_WAITING_PERIOD: 15m
         CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
         DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml

@@ -0,0 +1,53 @@
+agents:
+  provider: gcp
+  image: family/kibana-ubuntu-2004
+  imageProject: elastic-images-prod
+
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      machineType: n2-standard-16
+      preemptible: true
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 90
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/artifacts/docker_image.sh
+    label: 'Build Project Image'
+    key: build_project_image
+    agents:
+      machineType: n2-standard-16
+      preemptible: true
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - wait: ~
+
+  - command: .buildkite/scripts/steps/serverless/deploy.sh
+    label: 'Deploy Project'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    timeout_in_minutes: 10

.buildkite/pull_requests.json

@@ -30,7 +30,8 @@
         "^\\.backportrc\\.json$",
         "^nav-kibana-dev\\.docnav\\.json$",
         "^src/dev/prs/kibana_qa_pr_list\\.json$",
-        "^\\.buildkite/pull_requests\\.json$"
+        "^\\.buildkite/pull_requests\\.json$",
+        "^\\.devcontainer/"
       ],
       "always_require_ci_on_changed": [
         "^docs/developer/plugin-list.asciidoc$",
@@ -46,6 +47,30 @@
         "/__snapshots__/",
         "\\.test\\.(ts|tsx|js|jsx)"
       ]
+    },
+    {
+      "repoOwner": "elastic",
+      "repoName": "kibana",
+      "pipelineSlug": "kibana-deploy-project-from-pr",
+
+      "enabled": true,
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": false,
+      "build_drafts": false,
+      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:project))$",
+      "kibana_versions_check": true,
+      "kibana_build_reuse": true,
+      "kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge", "kibana-deploy-project-from-pr"],
+      "kibana_build_reuse_regexes": [
+        "^test/",
+        "^x-pack/test/",
+        "/__snapshots__/",
+        "\\.test\\.(ts|tsx|js|jsx)"
+      ]
     }
   ]
 }

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -20,8 +20,8 @@ if (!prConfig) {
 }
 
 const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
-const REQUIRED_PATHS = prConfig.always_require_ci_on_changed.map((r) => new RegExp(r, 'i'));
-const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed.map((r) => new RegExp(r, 'i'));
+const REQUIRED_PATHS = prConfig.always_require_ci_on_changed!.map((r) => new RegExp(r, 'i'));
+const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed!.map((r) => new RegExp(r, 'i'));
 
 const getPipeline = (filename: string, removeSteps = true) => {
   const str = fs.readFileSync(filename).toString();

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -51,7 +51,7 @@ fi
 if is_pr_with_label "ci:cloud-redeploy"; then
   echo "--- Shutdown Previous Deployment"
   CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
-  if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
+  if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" == "null" ]; then
     echo "No deployment to remove"
   else
     echo "Shutting down previous deployment..."

.buildkite/scripts/steps/functional/fleet_cypress.sh

@@ -12,4 +12,4 @@ echo "--- Fleet Cypress tests (Chrome)"
 cd x-pack/plugins/fleet
 
 set +e
-yarn cypress:run:reporter; status=$?; yarn junit:merge || :; exit $status
+yarn cypress:run:reporter; status=$?; yarn cypress_space_awareness:run:reporter; space_status=$?; yarn junit:merge || :; [ "$status" -ne 0 ] && exit $status || [ "$space_status" -ne 0 ] && exit $space_status || exit 0

.buildkite/scripts/steps/serverless/deploy.sh

@@ -56,7 +56,7 @@ deploy() {
 
   PROJECT_ID=$(jq -r '[.items[] | select(.name == "'$PROJECT_NAME'")] | .[0].id' $PROJECT_EXISTS_LOGS)
   if is_pr_with_label "ci:project-redeploy"; then
-    if [ -z "${PROJECT_ID}" ]; then
+    if [ -z "${PROJECT_ID}" ] || [ "${PROJECT_ID}" == "null" ]; then
       echo "No project to remove"
     else
       echo "Shutting down previous project..."
@@ -159,13 +159,13 @@ EOF
 }
 
 is_pr_with_label "ci:project-deploy-elasticsearch" && deploy "elasticsearch"
+is_pr_with_label "ci:project-deploy-security" && deploy "security"
 if is_pr_with_label "ci:project-deploy-observability" ; then
   # Only deploy observability if the PR is targeting main
   if [[ "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" == "main" ]]; then
     create_github_issue_oblt_test_environments
     buildkite-agent annotate --context obl-test-info --style info 'See linked [Deploy Serverless Kibana] issue in pull request for project deployment information'
   fi
 fi
-is_pr_with_label "ci:project-deploy-security" && deploy "security"
 
 exit 0;

.buildkite/scripts/steps/storybooks/build_and_upload.ts

@@ -43,6 +43,7 @@ const STORYBOOKS = [
   'lists',
   'observability',
   'observability_ai_assistant',
+  'observability_shared',
   'presentation',
   'security_solution',
   'security_solution_packages',

.eslintrc.js

@@ -974,6 +974,12 @@ module.exports = {
         '@kbn/telemetry/event_generating_elements_should_be_instrumented': 'error',
       },
     },
+    {
+      files: ['x-pack/plugins/search*/**/*.tsx', 'x-pack/packages/search/**/*.tsx'],
+      rules: {
+        '@kbn/telemetry/event_generating_elements_should_be_instrumented': 'warn',
+      },
+    },
     {
       files: [
         'x-pack/plugins/observability_solution/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)',
@@ -1815,9 +1821,23 @@ module.exports = {
       files: [
         'src/plugins/interactive_setup/**/*.{js,mjs,ts,tsx}',
         'test/interactive_setup_api_integration/**/*.{js,mjs,ts,tsx}',
+        'test/interactive_setup_functional/**/*.{js,mjs,ts,tsx}',
+
+        'packages/kbn-mock-idp-plugin/**/*.{js,mjs,ts,tsx}',
+        'packages/kbn-mock-idp-utils/**/*.{js,mjs,ts,tsx}',
+        'packages/kbn-security-hardening/**/*.{js,mjs,ts,tsx}',
+        'packages/kbn-user-profile-components/**/*.{js,mjs,ts,tsx}',
+
         'x-pack/plugins/encrypted_saved_objects/**/*.{js,mjs,ts,tsx}',
+        'x-pack/test/encrypted_saved_objects_api_integration/**/*.{js,mjs,ts,tsx}',
+
         'x-pack/plugins/security/**/*.{js,mjs,ts,tsx}',
+        'x-pack/packages/security/**/*.{js,mjs,ts,tsx}',
+        'x-pack/test/security_api_integration/**/*.{js,mjs,ts,tsx}',
+        'x-pack/test/security_functional/**/*.{js,mjs,ts,tsx}',
+
         'x-pack/plugins/spaces/**/*.{js,mjs,ts,tsx}',
+        'x-pack/test/spaces_api_integration/**/*.{js,mjs,ts,tsx}',
       ],
       rules: {
         '@typescript-eslint/consistent-type-imports': 1,

.github/CODEOWNERS

@@ -11,6 +11,7 @@ x-pack/test/alerting_api_integration/common/plugins/actions_simulators @elastic/
 packages/kbn-actions-types @elastic/response-ops
 src/plugins/advanced_settings @elastic/appex-sharedux @elastic/kibana-management
 x-pack/packages/kbn-ai-assistant @elastic/search-kibana
+x-pack/packages/kbn-ai-assistant-common @elastic/search-kibana
 src/plugins/ai_assistant_management/selection @elastic/obs-knowledge-team
 x-pack/packages/ml/aiops_change_point_detection @elastic/ml-ui
 x-pack/packages/ml/aiops_common @elastic/ml-ui
@@ -89,8 +90,9 @@ x-pack/plugins/cloud_integrations/cloud_full_story @elastic/kibana-core
 x-pack/test/cloud_integration/plugins/saml_provider @elastic/kibana-core
 x-pack/plugins/cloud_integrations/cloud_links @elastic/kibana-core
 x-pack/plugins/cloud @elastic/kibana-core
-x-pack/packages/kbn-cloud-security-posture @elastic/kibana-cloud-security-posture
-x-pack/packages/kbn-cloud-security-posture-common @elastic/kibana-cloud-security-posture
+x-pack/packages/kbn-cloud-security-posture/public @elastic/kibana-cloud-security-posture
+x-pack/packages/kbn-cloud-security-posture/common @elastic/kibana-cloud-security-posture
+x-pack/packages/kbn-cloud-security-posture/graph @elastic/kibana-cloud-security-posture
 x-pack/plugins/cloud_security_posture @elastic/kibana-cloud-security-posture
 packages/shared-ux/code_editor/impl @elastic/appex-sharedux
 packages/shared-ux/code_editor/mocks @elastic/appex-sharedux
@@ -1122,6 +1124,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 
 ### Observability Plugins
 
+
 # Observability AI Assistant
 x-pack/test/observability_ai_assistant_api_integration @elastic/obs-ai-assistant
 x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
@@ -1370,6 +1373,8 @@ x-pack/test/**/deployment_agnostic/ @elastic/appex-qa #temporarily to monitor te
 /x-pack/test_serverless/**/test_suites/common/saved_objects_management/ @elastic/kibana-core
 /x-pack/test_serverless/api_integration/test_suites/common/core/ @elastic/kibana-core
 /x-pack/test_serverless/api_integration/test_suites/**/telemetry/ @elastic/kibana-core
+/x-pack/test/functional/es_archives/cases/migrations/8.8.0 @elastic/response-ops
+
 #CC# /src/core/server/csp/ @elastic/kibana-core
 #CC# /src/plugins/saved_objects/ @elastic/kibana-core
 #CC# /x-pack/plugins/cloud/ @elastic/kibana-core
@@ -1434,6 +1439,7 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
 #CC# /x-pack/plugins/security/ @elastic/kibana-security
 
 # Response Ops team
+/x-pack/test/functional/es_archives/cases/default @elastic/response-ops
 /x-pack/test_serverless/api_integration/test_suites/observability/config.ts @elastic/response-ops
 /x-pack/test_serverless/api_integration/test_suites/observability/index.ts @elastic/response-ops
 /x-pack/test_serverless/functional/page_objects/svl_triggers_actions_ui_page.ts @elastic/response-ops
@@ -1467,8 +1473,10 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
 /x-pack/test/functional/es_archives/alerts_legacy @elastic/response-ops
 /x-pack/test/functional/es_archives/observability/alerts @elastic/response-ops
 /x-pack/test/functional/es_archives/actions @elastic/response-ops
-/x-pack/test/functional/es_archives/rules_scheduled_task_id  @elastic/response-ops
-/x-pack/test/functional/es_archives/alerting/8_2_0  @elastic/response-ops
+/x-pack/test/functional/es_archives/rules_scheduled_task_id @elastic/response-ops
+/x-pack/test/functional/es_archives/alerting/8_2_0 @elastic/response-ops
+/x-pack/test/functional/es_archives/cases/signals/default @elastic/response-ops
+/x-pack/test/functional/es_archives/cases/signals/hosts_users @elastic/response-ops
 
 # Enterprise Search
 /x-pack/test_serverless/functional/page_objects/svl_ingest_pipelines.ts @elastic/search-kibana
@@ -1522,6 +1530,7 @@ x-pack/test/api_integration/apis/management/index_management/inference_endpoints
 /x-pack/test_serverless/functional/test_suites/security/index.feature_flags.ts @elastic/security-solution
 /x-pack/test_serverless/functional/test_suites/security/index.ts @elastic/security-solution
 #CC# /x-pack/plugins/security_solution/ @elastic/security-solution
+/x-pack/test/functional/es_archives/cases/signals/duplicate_ids @elastic/response-ops
 
 # Security Solution OpenAPI bundles
 /x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_* @elastic/security-detection-rule-management
@@ -1672,6 +1681,11 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 /x-pack/plugins/stack_connectors/server/connector_types/gemini @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra
 /x-pack/plugins/stack_connectors/common/gemini @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra
 
+# Inference API
+/x-pack/plugins/stack_connectors/public/connector_types/inference @elastic/appex-ai-infra @elastic/security-generative-ai @elastic/obs-ai-assistant
+/x-pack/plugins/stack_connectors/server/connector_types/inference @elastic/appex-ai-infra @elastic/security-generative-ai @elastic/obs-ai-assistant
+/x-pack/plugins/stack_connectors/common/inference @elastic/appex-ai-infra @elastic/security-generative-ai @elastic/obs-ai-assistant
+
 ## Defend Workflows owner connectors
 /x-pack/plugins/stack_connectors/public/connector_types/sentinelone @elastic/security-defend-workflows
 /x-pack/plugins/stack_connectors/server/connector_types/sentinelone @elastic/security-defend-workflows
@@ -1828,6 +1842,7 @@ x-pack/plugins/osquery @elastic/security-defend-workflows
 /x-pack/plugins/fleet/public/components/cloud_security_posture @elastic/fleet @elastic/kibana-cloud-security-posture
 /x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/components/cloud_security_posture @elastic/fleet @elastic/kibana-cloud-security-posture
 /x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.* @elastic/fleet @elastic/kibana-cloud-security-posture
+/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/cloud_posture_third_party_support_callout.* @elastic/fleet @elastic/kibana-cloud-security-posture
 /x-pack/plugins/security_solution/public/cloud_security_posture @elastic/kibana-cloud-security-posture
 /x-pack/test/security_solution_cypress/cypress/e2e/explore/hosts/vulnerabilities_contextual_flyout.cy.ts @elastic/kibana-cloud-security-posture
 

.github/updatecli/values.d/ironbank.yml

@@ -0,0 +1,2 @@
+config:
+  - path: src/dev/build/tasks/os_packages/docker_generator/templates/ironbank

.github/updatecli/values.d/scm.yml

@@ -0,0 +1,11 @@
+scm:
+  enabled: true
+  owner: elastic
+  repository: kibana
+  branch: main
+  commitusingapi: true
+  # begin updatecli-compose policy values
+  user: kibanamachine
+  email: [email protected]
+  # end updatecli-compose policy values
+