[ML] AIOps: Log Rate Analysis embeddable

.backportrc.json

@@ -4,6 +4,7 @@
   "targetBranchChoices": [
     "main",
     "8.x",
+    "8.16",
     "8.15",
     "8.14",
     "8.13",
@@ -54,7 +55,7 @@
   ],
   "branchLabelMapping": {
     "^v9.0.0$": "main",
-    "^v8.16.0$": "8.x",
+    "^v8.17.0$": "8.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/ftr_oblt_stateful_configs.yml

@@ -3,9 +3,10 @@ disabled:
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config_run.ts
+  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -39,6 +39,9 @@ disabled:
   - x-pack/test/fleet_cypress/config.space_awareness.ts
   - x-pack/test/fleet_cypress/visual_config.ts
 
+  # Default http2 config to use for performance journeys
+  - x-pack/performance/configs/http2_config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts
@@ -246,6 +249,7 @@ enabled:
   - x-pack/test/functional/apps/ml/permissions/config.ts
   - x-pack/test/functional/apps/ml/short_tests/config.ts
   - x-pack/test/functional/apps/ml/stack_management_jobs/config.ts
+  - x-pack/test/functional/apps/ml/memory_usage/config.ts
   - x-pack/test/functional/apps/monitoring/config.ts
   - x-pack/test/functional/apps/painless_lab/config.ts
   - x-pack/test/functional/apps/remote_clusters/config.ts
@@ -317,6 +321,7 @@ enabled:
   - x-pack/test/security_api_integration/saml.http2.config.ts
   - x-pack/test/security_api_integration/saml_cloud.config.ts
   - x-pack/test/security_api_integration/chips.config.ts
+  - x-pack/test/security_api_integration/features.config.ts
   - x-pack/test/security_api_integration/session_idle.config.ts
   - x-pack/test/security_api_integration/session_invalidate.config.ts
   - x-pack/test/security_api_integration/session_lifespan.config.ts
@@ -340,10 +345,11 @@ enabled:
   - x-pack/test/task_manager_claimer_mget/config.ts
   - x-pack/test/ui_capabilities/security_and_spaces/config.ts
   - x-pack/test/ui_capabilities/spaces_only/config.ts
-  - x-pack/test/upgrade_assistant_integration/config.js
+  - x-pack/test/upgrade_assistant_integration/config.ts
   - x-pack/test/usage_collection/config.ts
   - x-pack/performance/journeys_e2e/aiops_log_rate_analysis.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard.ts
+  - x-pack/performance/journeys_e2e/ecommerce_dashboard_http2.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_map_only.ts
   - x-pack/performance/journeys_e2e/flight_dashboard.ts
   - x-pack/performance/journeys_e2e/login.ts
@@ -357,6 +363,7 @@ enabled:
   - x-pack/performance/journeys_e2e/web_logs_dashboard_esql.ts
   - x-pack/performance/journeys_e2e/web_logs_dashboard_dataview.ts
   - x-pack/performance/journeys_e2e/data_stress_test_lens.ts
+  - x-pack/performance/journeys_e2e/data_stress_test_lens_http2.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_saved_search_only.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_tsvb_gauge_only.ts
   - x-pack/performance/journeys_e2e/dashboard_listing_page.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -1,17 +1,17 @@
 disabled:
   # Base config files, only necessary to inform config finding script
-  - x-pack/test_serverless/functional/test_suites/security/cypress/security_config.base.ts
-  - x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.edr_workflows.ts
+  - x-pack/test/defend_workflows_cypress/serverless_config.base.ts
+  - x-pack/test/osquery_cypress/serverless_config.base.ts
 
   # Cypress configs, for now these are still run manually
   - x-pack/test/defend_workflows_cypress/serverless_config.ts
   - x-pack/test/osquery_cypress/serverless_cli_config.ts
-  - x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts
   - x-pack/test/security_solution_cypress/serverless_config.ts
 
+
   # Playwright
   - x-pack/test/security_solution_playwright/serverless_config.ts
 
@@ -98,6 +98,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   # serverless config files that run deployment-agnostic tests

.buildkite/ftr_security_stateful_configs.yml

@@ -69,10 +69,14 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/explore/network/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/explore/users/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/explore/overview/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/basic_license_essentials_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/ess.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/security_and_spaces/configs/ess.basic.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/security_and_spaces/configs/ess.trial.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/basic_license_essentials_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/ess.config.ts
@@ -81,6 +85,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts

.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.15 7.17
+      branch_configuration: main 8.x 8.16 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/build.yml
@@ -53,6 +53,10 @@ spec:
           cronline: 0 22 * * * America/New_York
           message: Daily build
           branch: '8.x'
+        Daily build (8.16):
+          cronline: 0 22 * * * America/New_York
+          message: Daily build
+          branch: '8.16'
         Daily build (8.15):
           cronline: 0 22 * * * America/New_York
           message: Daily build
@@ -88,7 +92,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.15 7.17
+      branch_configuration: main 8.x 8.16 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/promote.yml
@@ -138,7 +142,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.15 7.17
+      branch_configuration: main 8.x 8.16 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/verify.yml

.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-unsupported-ftrs-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.15 7.17
+      branch_configuration: main 8.x 8.16 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge_unsupported_ftrs.yml

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -25,7 +25,7 @@ spec:
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.15 7.17
+      branch_configuration: main 8.x 8.16 8.15 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge.yml

.buildkite/pipeline-resource-definitions/kibana-vm-images.yml

@@ -0,0 +1,48 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-vm-images
+  description: Build CI agent VM images for Kibana
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-vm-images'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / vm images
+      description: Build CI agent VM images for Kibana
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      default_branch: main
+      repository: elastic/ci-agent-images
+      pipeline_file: vm-images/.buildkite/pipeline.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        trigger_mode: none
+      schedules:
+        daily kibana image build:
+          branch: main
+          cronline: '0 0 * * *'
+          env:
+            IMAGES_CONFIG: kibana/images.yml
+          message: Builds Kibana VM images daily
+        daily kibana fips image build:
+          branch: main
+          cronline: '0 4 * * *' # make sure this runs after the daily kibana image build
+          env:
+            BASE_IMAGES_CONFIG: 'core/images.yml,kibana/images.yml'
+            IMAGES_CONFIG: kibana/fips.yml
+          message: Builds Kibana FIPS VM image daily
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -37,6 +37,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-vm-images.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/scalability_testing-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml

.buildkite/pipelines/chrome_forward_testing.yml

@@ -300,7 +300,7 @@ steps:
           limit: 1
 
   - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-    label: 'Serverless Osquery Cypress Tests'
+    label: 'Osquery Cypress Tests on Serverless'
     agents:
       machineType: n2-standard-4
       preemptible: true
@@ -345,17 +345,6 @@ steps:
         - exit_status: '-1'
           limit: 1
 
-  - command: .buildkite/scripts/steps/checks.sh
-    label: 'Checks'
-    agents:
-      machineType: n2-standard-2
-      preemptible: true
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
   - wait: ~
     continue_on_failure: true
 

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -218,7 +218,7 @@ steps:
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-        label: "Serverless Osquery Cypress Tests"
+        label: "Osquery Cypress Tests on Serverless"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004

.buildkite/pipelines/flaky_tests/groups.json

@@ -23,7 +23,7 @@
     {
       "key": "cypress/security_serverless_explore",
       "name": "[Serverless] Security Solution Explore - Cypress"
-    },   
+    },
     {
       "key": "cypress/security_solution_rule_management",
       "name": "Security Solution Rule Management - Cypress"
@@ -87,6 +87,18 @@
     {
       "key": "cypress/apm_cypress",
       "name": "APM - Cypress"
+    },
+    {
+      "key": "cypress/inventory_cypress",
+      "name": "Inventory - Cypress"
+    },
+    {
+      "key": "cypress/cloud_security_posture",
+      "name": "Cloud Security Posture - Cypress"
+    },
+    {
+      "key": "cypress/cloud_security_posture_serverless",
+      "name": "[Serverless] Cloud Security Posture - Cypress"
     }
   ]
-}
+}

.buildkite/pipelines/on_merge.yml

@@ -405,7 +405,7 @@ steps:
           limit: 1
 
   - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-    label: 'Serverless Osquery Cypress Tests'
+    label: 'Osquery Cypress Tests on Serverless'
     agents:
       image: family/kibana-ubuntu-2004
       imageProject: elastic-images-prod
@@ -532,6 +532,20 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/checks/capture_oas_snapshot.sh
+    label: 'Check OAS Snapshot'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'
     agents:

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -80,3 +80,20 @@ steps:
           limit: 3
         - exit_status: '*'
           limit: 1
+
+  - command: .buildkite/scripts/steps/functional/inventory_cypress.sh
+    label: 'Inventory Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pointer_compression.yml

@@ -339,7 +339,7 @@ steps:
           limit: 1
 
   - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-    label: 'Serverless Osquery Cypress Tests'
+    label: 'Osquery Cypress Tests on Serverless'
     agents:
       image: family/kibana-ubuntu-2004
       imageProject: elastic-images-prod

.buildkite/pipelines/pull_request/base.yml

@@ -109,6 +109,17 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/checks/capture_oas_snapshot.sh
+    label: 'Check OAS Snapshot'
+    agents:
+      machineType: n2-standard-2
+      preemptible: true
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh
     label: 'Build API Docs'
     agents: