Merge remote-tracking branch 'upstream/main' into rule-type-diff-algo…

…rithm

.buildkite/ftr_platform_stateful_configs.yml

@@ -351,6 +351,8 @@ enabled:
   - x-pack/performance/journeys_e2e/tsdb_logs_data_visualizer.ts
   - x-pack/performance/journeys_e2e/promotion_tracking_dashboard.ts
   - x-pack/performance/journeys_e2e/web_logs_dashboard.ts
+  - x-pack/performance/journeys_e2e/web_logs_dashboard_esql.ts
+  - x-pack/performance/journeys_e2e/web_logs_dashboard_dataview.ts
   - x-pack/performance/journeys_e2e/data_stress_test_lens.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_saved_search_only.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_tsvb_gauge_only.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -81,6 +81,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/explore/hosts/trial_license_complete_tier/configs/serverless.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -62,6 +62,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/explore/hosts/trial_license_complete_tier/configs/ess.config.ts

.buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts

@@ -38,15 +38,13 @@ async function main() {
 
   const preamble = locationFileLines.slice(0, 1);
 
-  // eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
   const locationObj = jsYaml.load(
     locationFileLines.slice(1).join('\n')
   ) as BackstageLocationResource;
   locationObj.spec.targets = pipelines.map(
     (fileName) => `${resourceDefinitionsBaseUrl}/${fileName}`
   );
 
-  // eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
   const locationYaml = jsYaml.dump(locationObj, { lineWidth: 400 });
 
   fs.writeFileSync(locationFile, `${preamble.join('\n')}\n${locationYaml}`);

.buildkite/pipeline-utils/agent_images.ts

@@ -7,7 +7,6 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { dump } from 'js-yaml';
 import { BuildkiteClient, BuildkiteCommandStep } from './buildkite';
 

.buildkite/pipeline-utils/buildkite/client.ts

@@ -10,7 +10,6 @@
 import axios, { AxiosInstance } from 'axios';
 import { execSync, ExecSyncOptions } from 'child_process';
 
-// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { dump } from 'js-yaml';
 
 import { parseLinkHeader } from './parse_link_header';

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -12,7 +12,6 @@ import * as Fs from 'fs';
 import * as globby from 'globby';
 import minimatch from 'minimatch';
 
-// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { load as loadYaml } from 'js-yaml';
 
 import { BuildkiteClient, BuildkiteStep } from '../buildkite';

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -126,15 +126,15 @@ steps:
       env:
         CONTAINER: "$KIBANA_IMAGE"
     soft_fail: true
-  - label: ":argo: Update kibana image tag for kibana-controller using gpctl"
+  - label: ":serverless::argo: Run synthetics tests and update kibana image tag to ${GIT_ABBREV_COMMIT} for kibana-controller"
     branches: main
-    trigger: gpctl-promote-with-e2e-tests
+    trigger: gpctl-promote-after-serverless-devenv-synthetics
     build:
       env:
         SERVICE_COMMIT_HASH: "$GIT_ABBREV_COMMIT"
         SERVICE: kibana
         REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/dev.yaml
-        DRY_RUN: "${DRY_RUN:-false}"
+        GPCTL_PROMOTE_DRY_RUN: ${DRY_RUN:-false}
 EOF
 
 else

.buildkite/scripts/steps/esql_generate_function_metadata.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 VALIDATION_PACKAGE_DIR="packages/kbn-esql-validation-autocomplete"
-EDITOR_PACKAGE_DIR="packages/kbn-text-based-editor"
+EDITOR_PACKAGE_DIR="packages/kbn-language-documentation"
 GIT_SCOPE="$VALIDATION_PACKAGE_DIR/**/* $EDITOR_PACKAGE_DIR/**/*"
 
 report_main_step () {

.buildkite/scripts/steps/openapi_bundling/final_merge.sh

@@ -7,3 +7,6 @@ source .buildkite/scripts/common/util.sh
 echo --- Merge Kibana OpenAPI specs
 
 (cd oas_docs && make api-docs && make api-docs-lint)
+(cd oas_docs && make api-docs-staging && make api-docs-lint)
+
+check_for_changed_files "make api-docs && make api-docs-staging" true

.buildkite/scripts/steps/storybooks/build_and_upload.ts

@@ -54,7 +54,7 @@ const STORYBOOKS = [
   'language_documentation_popover',
   'unified_search',
   'random_sampling',
-  'text_based_editor',
+  'esql_editor',
 ];
 
 const GITHUB_CONTEXT = 'Build and Publish Storybooks';

.devcontainer/Dockerfile

@@ -4,7 +4,6 @@ ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
 ENV HOME=/home/vscode
 ENV NVM_DIR=${HOME}/nvm
 ENV NVM_VERSION=v0.39.1
-ENV KBN_DIR=/workspaces/kibana
 ENV OPENSSL_PATH=${HOME}/openssl
 # Only specific versions are FIPS certified.
 ENV OPENSSL_VERSION='3.0.8'
@@ -50,6 +49,7 @@ WORKDIR ${KBN_DIR}
 # Node and NVM setup
 COPY .node-version /tmp/
 
+# Mac will have permissions issues if Node and NVM are installed as root
 USER vscode
 
 RUN mkdir -p $NVM_DIR && \

.devcontainer/devcontainer.json

@@ -23,8 +23,11 @@
     9230,
     9231
   ],
-  "postStartCommand": "/workspaces/kibana/.devcontainer/scripts/post_start.sh",
+  "postStartCommand": "${containerWorkspaceFolder}/.devcontainer/scripts/post_start.sh",
   "remoteUser": "vscode",
+  "containerEnv": {
+    "KBN_DIR": "${containerWorkspaceFolder}"
+  },
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker:2": {
       "version": "latest",

.github/CODEOWNERS

@@ -354,6 +354,7 @@ x-pack/plugins/data_quality @elastic/obs-ux-logs-team
 test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery
 packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
 packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore
+x-pack/plugins/data_usage @elastic/obs-ai-assistant @elastic/security-solution
 src/plugins/data_view_editor @elastic/kibana-data-discovery
 examples/data_view_field_editor_example @elastic/kibana-data-discovery
 src/plugins/data_view_field_editor @elastic/kibana-data-discovery
@@ -428,6 +429,7 @@ src/plugins/esql @elastic/kibana-esql
 packages/kbn-esql-ast @elastic/kibana-esql
 examples/esql_ast_inspector @elastic/kibana-esql
 src/plugins/esql_datagrid @elastic/kibana-esql
+packages/kbn-esql-editor @elastic/kibana-esql
 packages/kbn-esql-utils @elastic/kibana-esql
 packages/kbn-esql-validation-autocomplete @elastic/kibana-esql
 examples/esql_validation_example @elastic/kibana-esql
@@ -474,7 +476,7 @@ src/plugins/files_management @elastic/appex-sharedux
 src/plugins/files @elastic/appex-sharedux
 packages/kbn-find-used-node-modules @elastic/kibana-operations
 x-pack/plugins/fleet @elastic/fleet
-packages/kbn-flot-charts @elastic/kibana-operations
+packages/kbn-flot-charts @elastic/kibana-presentation @elastic/stack-monitoring
 x-pack/test/ui_capabilities/common/plugins/foo_plugin @elastic/kibana-security
 packages/kbn-formatters @elastic/obs-ux-logs-team
 src/plugins/ftr_apis @elastic/kibana-core
@@ -552,7 +554,7 @@ src/plugins/kibana_usage_collection @elastic/kibana-core
 src/plugins/kibana_utils @elastic/appex-sharedux
 x-pack/plugins/kubernetes_security @elastic/kibana-cloud-security-posture
 x-pack/packages/kbn-langchain @elastic/security-generative-ai
-packages/kbn-language-documentation-popover @elastic/kibana-esql
+packages/kbn-language-documentation @elastic/kibana-esql
 x-pack/examples/lens_config_builder_example @elastic/kibana-visualizations
 packages/kbn-lens-embeddable-utils @elastic/obs-ux-infra_services-team @elastic/kibana-visualizations
 packages/kbn-lens-formula-docs @elastic/kibana-visualizations
@@ -608,13 +610,15 @@ x-pack/packages/ml/data_view_utils @elastic/ml-ui
 x-pack/packages/ml/date_picker @elastic/ml-ui
 x-pack/packages/ml/date_utils @elastic/ml-ui
 x-pack/packages/ml/error_utils @elastic/ml-ui
+x-pack/packages/ml/field_stats_flyout @elastic/ml-ui
 x-pack/packages/ml/in_memory_table @elastic/ml-ui
 x-pack/packages/ml/is_defined @elastic/ml-ui
 x-pack/packages/ml/is_populated_object @elastic/ml-ui
 x-pack/packages/ml/kibana_theme @elastic/ml-ui
 x-pack/packages/ml/local_storage @elastic/ml-ui
 x-pack/packages/ml/nested_property @elastic/ml-ui
 x-pack/packages/ml/number_utils @elastic/ml-ui
+x-pack/packages/ml/parse_interval @elastic/ml-ui
 x-pack/plugins/ml @elastic/ml-ui
 x-pack/packages/ml/query_utils @elastic/ml-ui
 x-pack/packages/ml/random_sampler_utils @elastic/ml-ui
@@ -626,6 +630,7 @@ x-pack/packages/ml/time_buckets @elastic/ml-ui
 x-pack/packages/ml/trained_models_utils @elastic/ml-ui
 x-pack/packages/ml/ui_actions @elastic/ml-ui
 x-pack/packages/ml/url_state @elastic/ml-ui
+x-pack/packages/ml/validators @elastic/ml-ui
 packages/kbn-mock-idp-plugin @elastic/kibana-security
 packages/kbn-mock-idp-utils @elastic/kibana-security
 packages/kbn-monaco @elastic/appex-sharedux
@@ -918,7 +923,6 @@ test
 x-pack/test 
 x-pack/performance @elastic/appex-qa
 x-pack/examples/testing_embedded_lens @elastic/kibana-visualizations
-packages/kbn-text-based-editor @elastic/kibana-esql
 x-pack/examples/third_party_lens_navigation_prompt @elastic/kibana-visualizations
 x-pack/examples/third_party_vis_lens_example @elastic/kibana-visualizations
 x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations
@@ -1361,6 +1365,7 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
 # Kibana Platform Security
 /.github/codeql @elastic/kibana-security
 /.github/workflows/codeql.yml @elastic/kibana-security
+/.github/workflows/codeql-stats.yml @elastic/kibana-security
 /src/dev/eslint/security_eslint_rule_tests.ts @elastic/kibana-security
 /src/core/server/integration_tests/config/check_dynamic_config.test.ts @elastic/kibana-security
 /src/plugins/telemetry/server/config/telemetry_labels.ts @elastic/kibana-security
@@ -1416,6 +1421,15 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
 /x-pack/test_serverless/functional/test_suites/security/screenshot_creation/response_ops_docs @elastic/response-ops
 /x-pack/test_serverless/functional/test_suites/observability/screenshot_creation/response_ops_docs @elastic/response-ops
 /x-pack/test_serverless/api_integration/test_suites/common/alerting/ @elastic/response-ops
+/x-pack/test/functional/es_archives/action_task_params @elastic/response-ops
+/x-pack/test/functional/es_archives/actions @elastic/response-ops
+/x-pack/test/functional/es_archives/alerting @elastic/response-ops
+/x-pack/test/functional/es_archives/alerts @elastic/response-ops
+/x-pack/test/functional/es_archives/alerts_legacy @elastic/response-ops
+/x-pack/test/functional/es_archives/observability/alerts @elastic/response-ops
+/x-pack/test/functional/es_archives/actions @elastic/response-ops
+/x-pack/test/functional/es_archives/rules_scheduled_task_id  @elastic/response-ops
+/x-pack/test/functional/es_archives/alerting/8_2_0  @elastic/response-ops
 
 # Enterprise Search
 /x-pack/test/functional_enterprise_search/ @elastic/search-kibana
@@ -1676,6 +1690,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 /x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine @elastic/security-detection-engine
 /x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/rule_gaps.ts @elastic/security-detection-engine
 /x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists @elastic/security-detection-engine
+/x-pack/test/functional/es_archives/asset_criticality @elastic/security-detection-engine
 
 ## Security Threat Intelligence - Under Security Platform
 /x-pack/plugins/security_solution/public/common/components/threat_match @elastic/security-detection-engine

.github/workflows/codeql-stats.yml

@@ -0,0 +1,28 @@
+name: CodeQL statistics
+
+on:
+  schedule:
+    - cron: '27 0 * * 1' # At 00:27 every Monday
+
+jobs:
+  stats:
+    name: CodeQL statistics
+    runs-on: ubuntu-latest
+    if: github.repository == 'elastic/kibana' # Hack: Do not run on forks
+    steps:
+    - name: Checkout kibana-operations
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        repository: 'elastic/kibana-operations'
+        ref: main
+        path: ./kibana-operations
+        token: ${{secrets.KIBANAMACHINE_TOKEN}}
+
+    - name: CodeQL alert statistics
+      working-directory: ./kibana-operations/triage
+      env:
+        GITHUB_TOKEN: ${{secrets.KIBANAMACHINE_TOKEN}}
+        SLACK_TOKEN: ${{secrets.CODE_SCANNING_SLACK_TOKEN}}
+      run: |
+        npm ci --omit=dev
+        node codeql-alert-stats

.github/workflows/codeql.yml

@@ -76,12 +76,3 @@ jobs:
       run: |
         npm ci --omit=dev
         node codeql-alert 
-
-    - name: CodeQL alert statistics
-      working-directory: ./kibana-operations/triage
-      env:
-        GITHUB_TOKEN: ${{secrets.KIBANAMACHINE_TOKEN}}
-        SLACK_TOKEN: ${{secrets.CODE_SCANNING_SLACK_TOKEN}}
-      run: |
-        npm ci --omit=dev
-        node codeql-alert-stats

.github/workflows/on-merge.yml

@@ -3,26 +3,30 @@ on:
     types:
       - closed
       - labeled
+      - unlabeled
 
 jobs:
   on-merge:
     name: 'Label and Backport'
     runs-on: ubuntu-latest
     if: |
-      github.event.pull_request.merged == true
-      && (
-        (
-          github.event.action == 'labeled' && (
-            github.event.label.name == 'backport:prev-minor'
-            || github.event.label.name == 'backport:prev-major'
-            || github.event.label.name == 'backport:current-major'
-            || github.event.label.name == 'backport:all-open'
-            || github.event.label.name == 'backport:version'
-            || github.event.label.name == 'auto-backport'
-          )
-        )
-        || (github.event.action == 'closed')
-      )
+      github.event.pull_request.merged == true &&
+        (github.event.action == 'closed' ||
+          (github.event.action == 'labeled' &&
+            (github.event.label.name == 'backport:prev-minor' ||
+              github.event.label.name == 'backport:prev-major' ||
+              github.event.label.name == 'backport:current-major' ||
+              github.event.label.name == 'backport:all-open' ||
+              github.event.label.name == 'backport:version' ||
+              github.event.label.name == 'auto-backport')) ||
+          (github.event.action == 'unlabeled' &&
+            github.event.label.name == 'backport:skip' &&
+            (contains(github.event.pull_request.labels.*.name, 'backport:prev-minor') ||
+              contains(github.event.pull_request.labels.*.name, 'backport:prev-major') ||
+              contains(github.event.pull_request.labels.*.name, 'backport:current-major') ||
+              contains(github.event.pull_request.labels.*.name, 'backport:all-open') ||
+              contains(github.event.pull_request.labels.*.name, 'backport:version') ||
+              contains(github.event.pull_request.labels.*.name, 'auto-backport'))))
     steps:
       - name: Checkout Actions
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

.i18nrc.json

@@ -98,7 +98,7 @@
     "presentationUtil": "src/plugins/presentation_util",
     "randomSampling": "x-pack/packages/kbn-random-sampling",
     "reactPackages": "packages/react",
-    "textBasedEditor": "packages/kbn-text-based-editor",
+    "esqlEditor": "packages/kbn-esql-editor",
     "esqlUtils": "packages/kbn-esql-utils",
     "reporting": "packages/kbn-reporting",
     "savedObjects": "src/plugins/saved_objects",
@@ -118,7 +118,7 @@
     "serverlessPackages": "packages/serverless",
     "sse": [ "packages/kbn-sse-utils" ],
     "coloring": "packages/kbn-coloring/src",
-    "languageDocumentationPopover": "packages/kbn-language-documentation-popover/src",
+    "languageDocumentation": "packages/kbn-language-documentation/src",
     "esql": "src/plugins/esql",
     "esqlDataGrid": "src/plugins/esql_datagrid",
     "statusPage": "src/legacy/core_plugins/status_page",

api_docs/actions.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/actions
 title: "actions"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the actions plugin
-date: 2024-09-18
+date: 2024-09-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
 ---
 import actionsObj from './actions.devdocs.json';

api_docs/advanced_settings.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/advancedSettings
 title: "advancedSettings"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the advancedSettings plugin
-date: 2024-09-18
+date: 2024-09-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
 ---
 import advancedSettingsObj from './advanced_settings.devdocs.json';

api_docs/ai_assistant_management_selection.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiAssistantManagementSelection
 title: "aiAssistantManagementSelection"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiAssistantManagementSelection plugin
-date: 2024-09-18
+date: 2024-09-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiAssistantManagementSelection']
 ---
 import aiAssistantManagementSelectionObj from './ai_assistant_management_selection.devdocs.json';

api_docs/aiops.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiops
 title: "aiops"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiops plugin
-date: 2024-09-18
+date: 2024-09-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops']
 ---
 import aiopsObj from './aiops.devdocs.json';