Merge branch 'main' into d/2024-07-23-favorites

.buildkite/ftr_base_serverless_configs.yml

@@ -1,6 +1,8 @@
 disabled:
   # Base config files, only necessary to inform config finding script
 
+  # Serverless deployment-agnostic default config for api-integration tests
+  - x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts
   # Serverless base config files
   - x-pack/test_serverless/api_integration/config.base.ts
   - x-pack/test_serverless/functional/config.base.ts

.buildkite/ftr_oblt_serverless_configs.yml

@@ -26,3 +26,5 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/oblt.serverless.config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -1,4 +1,6 @@
 disabled:
+  # Stateful base config for deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/default_configs/stateful.config.base.ts
   # Base config files, only necessary to inform config finding script
   - test/functional/config.base.js
   - test/functional/firefox/config.base.ts
@@ -155,7 +157,6 @@ enabled:
   - x-pack/test/api_integration/apis/monitoring/config.ts
   - x-pack/test/api_integration/apis/monitoring_collection/config.ts
   - x-pack/test/api_integration/apis/osquery/config.ts
-  - x-pack/test/api_integration/apis/painless_lab/config.ts
   - x-pack/test/api_integration/apis/search/config.ts
   - x-pack/test/api_integration/apis/searchprofiler/config.ts
   - x-pack/test/api_integration/apis/security/config.ts
@@ -359,3 +360,5 @@ enabled:
   - x-pack/performance/journeys_e2e/apm_service_inventory.ts
   - x-pack/performance/journeys_e2e/infra_hosts_view.ts
   - x-pack/test/custom_branding/config.ts
+  # stateful config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/stateful.config.ts

.buildkite/ftr_search_serverless_configs.yml

@@ -16,3 +16,5 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/search.serverless.config.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -97,3 +97,5 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/security.serverless.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -83,4 +83,5 @@ enabled:
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts
   - x-pack/test/cloud_security_posture_api/config.ts
   - x-pack/test/cloud_security_posture_functional/config.ts
+  - x-pack/test/cloud_security_posture_functional/config.agentless.ts
   - x-pack/test/cloud_security_posture_functional/data_views/config.ts

.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml

@@ -19,12 +19,12 @@ spec:
       description: Run Kibana FIPS smoke tests
     spec:
       env:
-        SLACK_NOTIFICATIONS_CHANNEL: "#kibana-fips-ftr-errors"
-        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       repository: elastic/kibana
       branch_configuration: main
       default_branch: main
-      pipeline_file: ".buildkite/pipelines/fips.yml"
+      pipeline_file: '.buildkite/pipelines/fips.yml'
       provider_settings:
         trigger_mode: none
       schedules:

.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# renovate.json tracks this file for updating the pipelib docker image, update the path if you move this file
 # This script is used to validate a single RRE for a pipeline definition.
 
 TARGET_FILE=$1
@@ -16,7 +17,7 @@ FOLDER_NAME=$(dirname "$ABSOLUTE_PATH")
 
 docker run -it \
  --mount type=bind,source="$FOLDER_NAME",target=/home/app/ \
-  docker.elastic.co/ci-agent-images/pipelib \
+  docker.elastic.co/ci-agent-images/pipelib:0.8.0@sha256:641d7fc6cfe473900a1fbe49876762916d804b09fdf2945f74e9f803f3073779 \
   rre validate --backstage-entity-aware "/home/app/$FILE_NAME"
 
 if [ $? -ne 0 ]; then

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -44,26 +44,26 @@ steps:
         timeout_in_minutes: 60
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 3
 
       - label: "Pick Test Group Run Order (FTR + Integration)"
         command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
         depends_on: build
         timeout_in_minutes: 10
         env:
-          FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh'
-          JEST_INTEGRATION_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/jest_integration.sh'
-          FTR_CONFIG_PATTERNS: '**/test_serverless/**'
-          FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'
-          LIMIT_CONFIG_TYPE: 'functional,integration'
+          FTR_CONFIGS_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh"
+          JEST_INTEGRATION_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/jest_integration.sh"
+          FTR_CONFIG_PATTERNS: "**/test_serverless/**,**/test/security_solution_api_integration/**/serverless.config.ts"
+          FTR_EXTRA_ARGS: "$FTR_EXTRA_ARGS"
+          LIMIT_CONFIG_TYPE: "functional,integration"
         retry:
           automatic:
-            - exit_status: '*'
+            - exit_status: "*"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh
-        label: 'Serverless Entity Analytics - Security Solution Cypress Tests'
+        label: "Serverless Entity Analytics - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -76,11 +76,11 @@ steps:
         parallelism: 3
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
-        label: 'Serverless Explore - Security Solution Cypress Tests'
+        label: "Serverless Explore - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -93,11 +93,11 @@ steps:
         parallelism: 4
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
-        label: 'Serverless Investigations - Security Solution Cypress Tests'
+        label: "Serverless Investigations - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -110,11 +110,11 @@ steps:
         parallelism: 8
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
-        label: 'Serverless Rule Management - Security Solution Cypress Tests'
+        label: "Serverless Rule Management - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -127,11 +127,11 @@ steps:
         parallelism: 5
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
-        label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+        label: "Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -144,11 +144,11 @@ steps:
         parallelism: 1
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
-        label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+        label: "Serverless Detection Engine - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -161,11 +161,11 @@ steps:
         parallelism: 5
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
-        label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+        label: "Serverless Detection Engine - Exceptions - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -178,11 +178,11 @@ steps:
         parallelism: 2
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
-        label: 'Serverless AI Assistant - Security Solution Cypress Tests'
+        label: "Serverless AI Assistant - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -195,11 +195,11 @@ steps:
         parallelism: 1
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
-        label: 'Defend Workflows Cypress Tests on Serverless'
+        label: "Defend Workflows Cypress Tests on Serverless"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -214,11 +214,11 @@ steps:
         parallelism: 12
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-        label: 'Serverless Osquery Cypress Tests'
+        label: "Serverless Osquery Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -231,7 +231,7 @@ steps:
         parallelism: 7
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
   - wait: ~
@@ -241,6 +241,6 @@ steps:
 
   - wait: ~
 
-  - label: 'Post-Build'
+  - label: "Post-Build"
     command: .buildkite/scripts/lifecycle/post_build.sh
     timeout_in_minutes: 10

.buildkite/pipelines/sonarqube.yml

@@ -2,8 +2,8 @@ env:
   SKIP_NODE_SETUP: true
 
 steps:
-  - label: ":sonarqube: Continuous Code Inspection"
+  - label: ':sonarqube: Continuous Code Inspection'
     agents:
       image: docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest
-      memory: 16G
+      memory: 32G
     command: /scan-source-code.sh

.buildkite/scripts/common/env.sh

@@ -23,12 +23,14 @@ if [[ -d /opt/local-ssd/buildkite ]]; then
   mkdir -p "$TMPDIR"
 fi
 
-KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
-export KIBANA_PKG_BRANCH
-export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
+if command -v jq >/dev/null 2>&1; then
+  KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
+  export KIBANA_PKG_BRANCH
+  export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
 
-KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
-export KIBANA_PKG_VERSION
+  KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
+  export KIBANA_PKG_VERSION
+fi
 
 # Detects and exports the final target branch when using a merge queue
 if [[ "${BUILDKITE_BRANCH:-}" == "gh-readonly-queue"* ]]; then

.buildkite/scripts/steps/checks/renovate.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# renovate.json tracks this file for updating the pipelib docker image, update the path if you move this file
+
+set -euo pipefail
+
+echo --- Check renovate.json
+docker run -v "$(pwd)"/renovate.json:/home/app/renovate.json docker.elastic.co/ci-agent-images/pipelib:0.8.0@sha256:641d7fc6cfe473900a1fbe49876762916d804b09fdf2945f74e9f803f3073779 renovate-config-validator

.buildkite/scripts/steps/quick_checks.sh

@@ -23,3 +23,4 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/ftr_configs.sh
 .buildkite/scripts/steps/checks/yarn_deduplicate.sh
 .buildkite/scripts/steps/checks/prettier_topology.sh
+.buildkite/scripts/steps/checks/renovate.sh

.eslintrc.js

@@ -618,7 +618,7 @@ module.exports = {
         'test/*/*.config.ts',
         'test/*/{tests,test_suites,apis,apps}/**/*',
         'test/server_integration/**/*.ts',
-        'x-pack/test/*/{tests,test_suites,apis,apps}/**/*',
+        'x-pack/test/*/{tests,test_suites,apis,apps,deployment_agnostic}/**/*',
         'x-pack/test/*/*config.*ts',
         'x-pack/test/saved_object_api_integration/*/apis/**/*',
         'x-pack/test/ui_capabilities/*/tests/**/*',

.github/CODEOWNERS

@@ -50,7 +50,6 @@ packages/kbn-apm-types @elastic/obs-ux-infra_services-team
 packages/kbn-apm-utils @elastic/obs-ux-infra_services-team
 test/plugin_functional/plugins/app_link_test @elastic/kibana-core
 x-pack/test/usage_collection/plugins/application_usage_test @elastic/kibana-core
-x-pack/plugins/observability_solution/assets_data_access @elastic/obs-knowledge-team
 x-pack/test/security_api_integration/plugins/audit_log @elastic/kibana-security
 packages/kbn-avc-banner @elastic/security-defend-workflows
 packages/kbn-axe-config @elastic/kibana-qa
@@ -394,6 +393,7 @@ src/plugins/embeddable @elastic/kibana-presentation
 x-pack/examples/embedded_lens_example @elastic/kibana-visualizations
 x-pack/plugins/encrypted_saved_objects @elastic/kibana-security
 x-pack/plugins/enterprise_search @elastic/search-kibana
+x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities
 x-pack/packages/kbn-entities-schema @elastic/obs-entities
 x-pack/plugins/observability_solution/entity_manager @elastic/obs-entities
 examples/error_boundary @elastic/appex-sharedux
@@ -778,6 +778,7 @@ packages/kbn-securitysolution-t-grid @elastic/security-detection-engine
 packages/kbn-securitysolution-utils @elastic/security-detection-engine
 packages/kbn-server-http-tools @elastic/kibana-core
 packages/kbn-server-route-repository @elastic/obs-knowledge-team
+packages/kbn-server-route-repository-client @elastic/obs-knowledge-team
 packages/kbn-server-route-repository-utils @elastic/obs-knowledge-team
 x-pack/plugins/serverless @elastic/appex-sharedux
 packages/serverless/settings/common @elastic/appex-sharedux @elastic/kibana-management
@@ -864,6 +865,7 @@ packages/kbn-stdio-dev-helpers @elastic/kibana-operations
 packages/kbn-storybook @elastic/kibana-operations
 x-pack/plugins/observability_solution/synthetics/e2e @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/synthetics @elastic/obs-ux-management-team
+x-pack/packages/kbn-synthetics-private-location @elastic/obs-ux-management-team
 x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture @elastic/response-ops
 x-pack/test/plugin_api_perf/plugins/task_manager_performance @elastic/response-ops
 x-pack/plugins/task_manager @elastic/response-ops
@@ -1276,6 +1278,10 @@ x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
 /x-pack/test_serverless/functional/test_suites/security/ftr/ @elastic/appex-qa
 /x-pack/test_serverless/functional/test_suites/common/home_page/  @elastic/appex-qa
 /x-pack/test_serverless/**/services/ @elastic/appex-qa
+/packages/kbn-es/src/stateful_resources/roles.yml @elastic/appex-qa
+x-pack/test/api_integration/deployment_agnostic/default_configs/ @elastic/appex-qa
+x-pack/test/api_integration/deployment_agnostic/services/ @elastic/appex-qa
+x-pack/test/**/deployment_agnostic/ @elastic/appex-qa #temporarily to monitor tests migration
 
 # Core
 /config/ @elastic/kibana-core
@@ -1522,18 +1528,18 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 
 ## Generative AI owner connectors
 # OpenAI
-/x-pack/plugins/stack_connectors/public/connector_types/openai @elastic/security-generative-ai @elastic/obs-knowledge-team
-/x-pack/plugins/stack_connectors/server/connector_types/openai @elastic/security-generative-ai @elastic/obs-knowledge-team
-/x-pack/plugins/stack_connectors/common/openai @elastic/security-generative-ai @elastic/obs-knowledge-team
+/x-pack/plugins/stack_connectors/public/connector_types/openai @elastic/security-generative-ai @elastic/obs-ai-assistant
+/x-pack/plugins/stack_connectors/server/connector_types/openai @elastic/security-generative-ai @elastic/obs-ai-assistant
+/x-pack/plugins/stack_connectors/common/openai @elastic/security-generative-ai @elastic/obs-ai-assistant
 # Bedrock
-/x-pack/plugins/stack_connectors/public/connector_types/bedrock @elastic/security-generative-ai
-/x-pack/plugins/stack_connectors/server/connector_types/bedrock @elastic/security-generative-ai
-/x-pack/plugins/stack_connectors/common/bedrock @elastic/security-generative-ai
+/x-pack/plugins/stack_connectors/public/connector_types/bedrock @elastic/security-generative-ai @elastic/obs-ai-assistant
+/x-pack/plugins/stack_connectors/server/connector_types/bedrock @elastic/security-generative-ai @elastic/obs-ai-assistant
+/x-pack/plugins/stack_connectors/common/bedrock @elastic/security-generative-ai @elastic/obs-ai-assistant
 
 # Gemini
-/x-pack/plugins/stack_connectors/public/connector_types/gemini @elastic/security-generative-ai
-/x-pack/plugins/stack_connectors/server/connector_types/gemini @elastic/security-generative-ai
-/x-pack/plugins/stack_connectors/common/gemini @elastic/security-generative-ai
+/x-pack/plugins/stack_connectors/public/connector_types/gemini @elastic/security-generative-ai @elastic/obs-ai-assistant
+/x-pack/plugins/stack_connectors/server/connector_types/gemini @elastic/security-generative-ai @elastic/obs-ai-assistant
+/x-pack/plugins/stack_connectors/common/gemini @elastic/security-generative-ai @elastic/obs-ai-assistant
 
 ## Defend Workflows owner connectors
 /x-pack/plugins/stack_connectors/public/connector_types/sentinelone @elastic/security-defend-workflows

.github/workflows/auto-approve-api-docs.yml

@@ -15,4 +15,4 @@ jobs:
     permissions:
       pull-requests: write
     steps:
-      - uses: hmarr/auto-approve-action@f0939ea97e9205ef24d872e76833fa908a770363 #4.0.0
+      - uses: hmarr/auto-approve-action@f0939ea97e9205ef24d872e76833fa908a770363 # v4.0.0