Merge branch 'main' into fix/202266-2

.buildkite/package.json

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@octokit/rest": "^18.10.0",
-    "axios": "^1.7.4",
+    "axios": "^1.7.9",
     "globby": "^11.1.0",
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",
@@ -24,7 +24,7 @@
     "@types/mocha": "^10.0.1",
     "@types/node": "^15.12.2",
     "chai": "^4.3.10",
-    "mocha": "^10.8.2",
+    "mocha": "^11.0.1",
     "nock": "^12.0.2",
     "ts-node": "^10.9.2",
     "typescript": "^5.1.6"
@@ -33,4 +33,4 @@
     "#pipeline-utils": "./pipeline-utils/index.ts",
     "#pipeline-utils/*": "./pipeline-utils/*"
   }
-}
+}

.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml

@@ -0,0 +1,54 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-gen-ai-daily
+  description: Runs the GenAI FTR tests daily
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-gen-ai-tests-daily'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:appex-ai-infra'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / gen-ai-tests / daily
+      description: Runs the GenAI FTR tests daily
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#appex-ai-infra-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main
+      cancel_intermediate_builds: true
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/gen_ai_testing.yml
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        appex-ai-infra:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build:
+          cronline: 0 0 * * * America/New_York
+          message: Daily build
+          branch: main
+      tags:
+        - kibana

.buildkite/pipeline-resource-definitions/locations.yml

@@ -50,3 +50,4 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml

.buildkite/pipelines/gen_ai_testing.yml

@@ -0,0 +1,48 @@
+env:
+  FTR_GEN_AI: "1"
+steps:
+  - label: '👨‍🔧 Pre-Build'
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+
+  - wait
+
+  - label: '🧑‍🏭 Build Kibana Distribution'
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - wait
+
+  - command: .buildkite/scripts/steps/test/ftr_configs.sh
+    env:
+      FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts"
+      FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
+      FTR_GEN_AI: "1"
+    label: AppEx AI-Infra Inference APIs FTR tests
+    key: ai-infra-gen-ai-inference-api
+    timeout_in_minutes: 50
+    parallelism: 1
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+

.buildkite/pipelines/on_merge.yml

@@ -169,28 +169,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/ftr_configs.sh
-    env:
-      FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts"
-      FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
-      FTR_GEN_AI: "1"
-    label: AppEx AI-Infra Inference APIs FTR tests
-    key: ai-infra-gen-ai-inference-api
-    timeout_in_minutes: 50
-    parallelism: 1
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh
     label: 'Serverless Entity Analytics - Security Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/security_solution/automatic_import.yml

@@ -0,0 +1,20 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/security_serverless_automatic_import.sh
+    label: 'Serverless Automatic Import - Security Solution Cypress Tests'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on:
+      - build
+      - quick_checks
+      - checks
+      - linting
+      - linting_with_types
+      - check_types
+      - check_oas_snapshot
+    timeout_in_minutes: 60
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -266,6 +266,9 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(
         getPipeline('.buildkite/pipelines/pull_request/security_solution/ai_assistant.yml')
       );
+      pipeline.push(
+        getPipeline('.buildkite/pipelines/pull_request/security_solution/automatic_import.yml')
+      );
       pipeline.push(
         getPipeline('.buildkite/pipelines/pull_request/security_solution/detection_engine.yml')
       );

.buildkite/scripts/steps/esql_generate_function_metadata.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-VALIDATION_PACKAGE_DIR="packages/kbn-esql-validation-autocomplete"
-EDITOR_PACKAGE_DIR="packages/kbn-language-documentation"
+VALIDATION_PACKAGE_DIR="src/platform/packages/shared/kbn-esql-validation-autocomplete"
+EDITOR_PACKAGE_DIR="src/platform/packages/private/kbn-language-documentation"
 GIT_SCOPE="$VALIDATION_PACKAGE_DIR/**/* $EDITOR_PACKAGE_DIR/**/*"
 
 report_main_step () {

.buildkite/scripts/steps/esql_grammar_sync.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 synchronize_lexer_grammar () {
   license_header="$1"
   source_file="$PARENT_DIR/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4"
-  destination_file="./packages/kbn-esql-ast/src/antlr/esql_lexer.g4"
+  destination_file="./src/platform/packages/shared/kbn-esql-ast/src/antlr/esql_lexer.g4"
 
   # Copy the file
   cp "$source_file" "$destination_file"
@@ -26,7 +26,7 @@ synchronize_lexer_grammar () {
 synchronize_parser_grammar () {
   license_header="$1"
   source_file="$PARENT_DIR/elasticsearch/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4"
-  destination_file="./packages/kbn-esql-ast/src/antlr/esql_parser.g4"
+  destination_file="./src/platform/packages/shared/kbn-esql-ast/src/antlr/esql_parser.g4"
 
   # Copy the file
   cp "$source_file" "$destination_file"
@@ -104,7 +104,7 @@ main () {
   .buildkite/scripts/bootstrap.sh
 
   # Build ANTLR stuff
-  cd ./packages/kbn-esql-ast/src
+  cd ./src/platform/packages/shared/kbn-esql-ast/src
   yarn build:antlr4:esql
 
   # Make a commit

.buildkite/scripts/steps/functional/security_serverless_automatic_import.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Automatic Import Cypress Tests on Serverless"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:automatic_import:run:serverless; status=$?; yarn junit:merge || :; exit $status

.eslintignore

@@ -39,7 +39,7 @@ snapshots.js
 /packages/kbn-ui-framework/dist
 /packages/kbn-flot-charts/lib
 /packages/kbn-monaco/src/**/antlr
-/packages/kbn-esql-ast/src/**/antlr
+/src/platform/packages/shared/kbn-esql-ast/src/**/antlr
 
 # Bazel
 /bazel-*

.eslintrc.js

@@ -189,7 +189,7 @@ const DEV_PATTERNS = [
   'x-pack/performance/**/*',
   'src/setup_node_env/index.js',
   'src/cli/dev.js',
-  'packages/kbn-esql-validation-autocomplete/scripts/**/*',
+  'src/platform/packages/shared/kbn-esql-validation-autocomplete/scripts/**/*',
 ];
 
 /** Restricted imports with suggested alternatives */
@@ -967,7 +967,7 @@ module.exports = {
     },
     {
       files: [
-        'x-pack/plugins/aiops/**/*.tsx',
+        'x-pack/platform/plugins/shared/aiops/**/*.tsx',
         'x-pack/plugins/observability_solution/**/*.tsx',
         'x-pack/plugins/{streams,streams_app}/**/*.{ts,tsx}',
         'src/plugins/ai_assistant_management/**/*.tsx',
@@ -1073,11 +1073,12 @@ module.exports = {
      */
     {
       files: [
-        'x-pack/plugins/aiops/**/*.{js,mjs,ts,tsx}',
-        'x-pack/plugins/data_visualizer/**/*.{js,mjs,ts,tsx}',
-        'x-pack/plugins/ml/**/*.{js,mjs,ts,tsx}',
-        'x-pack/plugins/transform/**/*.{js,mjs,ts,tsx}',
-        'x-pack/packages/ml/**/*.{js,mjs,ts,tsx}',
+        'x-pack/platform/plugins/shared/aiops/**/*.{js,mjs,ts,tsx}',
+        'x-pack/platform/plugins/private/data_visualizer/**/*.{js,mjs,ts,tsx}',
+        'x-pack/platform/plugins/shared/ml/**/*.{js,mjs,ts,tsx}',
+        'x-pack/platform/plugins/private/transform/**/*.{js,mjs,ts,tsx}',
+        'x-pack/packages/private/ml/**/*.{js,mjs,ts,tsx}',
+        'x-pack/packages/shared/ml/**/*.{js,mjs,ts,tsx}',
       ],
       rules: {
         '@typescript-eslint/consistent-type-imports': 'error',