Merge branch 'main' into file-upload-filter-running-inference-services

.buildkite/ftr_oblt_stateful_configs.yml

@@ -3,9 +3,10 @@ disabled:
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config_run.ts
+  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts

.buildkite/pipelines/flaky_tests/groups.json

@@ -23,7 +23,7 @@
     {
       "key": "cypress/security_serverless_explore",
       "name": "[Serverless] Security Solution Explore - Cypress"
-    },   
+    },
     {
       "key": "cypress/security_solution_rule_management",
       "name": "Security Solution Rule Management - Cypress"
@@ -87,6 +87,10 @@
     {
       "key": "cypress/apm_cypress",
       "name": "APM - Cypress"
+    },
+    {
+      "key": "cypress/inventory_cypress",
+      "name": "Inventory - Cypress"
     }
   ]
-}
+}

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -80,3 +80,20 @@ steps:
           limit: 3
         - exit_status: '*'
           limit: 1
+
+  - command: .buildkite/scripts/steps/functional/inventory_cypress.sh
+    label: 'Inventory Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/inventory_cypress.yml

@@ -0,0 +1,17 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/inventory_cypress.sh
+    label: 'Inventory Cypress Tests'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on:
+      - build
+      - quick_checks
+    timeout_in_minutes: 120
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pull_requests.json

@@ -20,15 +20,12 @@
       "skip_target_branches": ["6.8", "7.11", "7.12"],
       "enable_skippable_commits": true,
       "skip_ci_on_only_changed": [
-        "^dev_docs/",
         "^docs/",
         "^rfcs/",
         "^\\.github/",
         "\\.md$",
-        "\\.mdx$",
         "^api_docs/.+\\.devdocs\\.json$",
         "^\\.backportrc\\.json$",
-        "^nav-kibana-dev\\.docnav\\.json$",
         "^src/dev/prs/kibana_qa_pr_list\\.json$",
         "^\\.buildkite/pull_requests\\.json$",
         "^\\.devcontainer/"

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -77,6 +77,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^x-pack\/plugins\/observability_solution\/inventory/,
+        /^packages\/kbn-apm-synthtrace/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/inventory_cypress.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([
         /^x-pack\/plugins\/observability_solution\/observability_onboarding/,

.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

@@ -69,3 +69,12 @@ cat << EOT | buildkite-agent annotate --style "success"
   <br/>Kibana commit: <a href="https://github.com/elastic/kibana/commit/$BUILDKITE_COMMIT">$BUILDKITE_COMMIT</a>
   <br/>Elasticsearch commit: <a href="https://github.com/elastic/elasticsearch/commit/$ELASTIC_COMMIT_HASH">$ELASTIC_COMMIT_HASH</a>
 EOT
+
+cat << EOF | buildkite-agent pipeline upload
+steps:
+  - label: "Builds Kibana VM images for cache update"
+    trigger: kibana-vm-images
+    build:
+      env:
+        IMAGES_CONFIG="kibana/images.yml"
+EOF

.buildkite/scripts/steps/es_snapshots/promote.sh

@@ -19,7 +19,7 @@ if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
   cat << EOF | buildkite-agent pipeline upload
 steps:
   - label: "Builds Kibana VM images for cache update"
-    trigger: ci-vm-images
+    trigger: kibana-vm-images
     build:
       env:
         IMAGES_CONFIG="kibana/images.yml"

.buildkite/scripts/steps/functional/inventory_cypress.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+export JOB=kibana-inventory-onboarding-cypress
+
+echo "--- Observability Inventory Cypress Tests"
+
+cd "$XPACK_DIR"
+
+node plugins/observability_solution/inventory/scripts/test/e2e.js \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

.github/CODEOWNERS

@@ -534,6 +534,7 @@ x-pack/plugins/integration_assistant @elastic/security-scalability
 src/plugins/interactive_setup @elastic/kibana-security
 test/interactive_setup_api_integration/plugins/test_endpoints @elastic/kibana-security
 packages/kbn-interpreter @elastic/kibana-visualizations
+x-pack/plugins/observability_solution/inventory/e2e @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/inventory @elastic/obs-ux-infra_services-team
 x-pack/plugins/observability_solution/investigate_app @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/investigate @elastic/obs-ux-management-team

config/serverless.es.yml

@@ -17,7 +17,7 @@ xpack.osquery.enabled: false
 xpack.fleet.enabled: true
 xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false
 xpack.fleet.internal.registry.spec.min: '3.0'
-xpack.fleet.internal.registry.spec.max: '3.2'
+xpack.fleet.internal.registry.spec.max: '3.3'
 xpack.fleet.packages:
   # fleet_server package installed to publish agent metrics
   - name: fleet_server

package.json

@@ -1443,6 +1443,7 @@
     "@kbn/get-repo-files": "link:packages/kbn-get-repo-files",
     "@kbn/import-locator": "link:packages/kbn-import-locator",
     "@kbn/import-resolver": "link:packages/kbn-import-resolver",
+    "@kbn/inventory-e2e": "link:x-pack/plugins/observability_solution/inventory/e2e",
     "@kbn/jest-serializers": "link:packages/kbn-jest-serializers",
     "@kbn/journeys": "link:packages/kbn-journeys",
     "@kbn/json-ast": "link:packages/kbn-json-ast",

packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts

@@ -15,29 +15,28 @@ class ContainerEntity extends Serializable<EntityFields> {
     super({
       ...fields,
       'entity.type': 'container',
-      'entity.definitionId': 'latest',
+      'entity.definitionId': 'builtin_containers_from_ecs_data',
+      'entity.identityFields': ['container.id'],
     });
   }
 }
 
 export function containerEntity({
   agentName,
   dataStreamType,
-  dataStreamDataset,
   containerId,
   entityId,
 }: {
   agentName: string[];
   dataStreamType: EntityDataStreamType[];
-  dataStreamDataset: string;
   containerId: string;
   entityId: string;
 }) {
   return new ContainerEntity({
     'source_data_stream.type': dataStreamType,
-    'source_data_stream.dataset': dataStreamDataset,
     'agent.name': agentName,
     'container.id': containerId,
+    'entity.displayName': containerId,
     'entity.id': entityId,
   });
 }

packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts

@@ -15,29 +15,28 @@ class HostEntity extends Serializable<EntityFields> {
     super({
       ...fields,
       'entity.type': 'host',
-      'entity.definitionId': 'latest',
+      'entity.definitionId': 'builtin_hosts_from_ecs_data',
+      'entity.identityFields': ['host.name'],
     });
   }
 }
 
 export function hostEntity({
   agentName,
   dataStreamType,
-  dataStreamDataset,
   hostName,
   entityId,
 }: {
   agentName: string[];
   dataStreamType: EntityDataStreamType[];
-  dataStreamDataset: string;
   hostName: string;
   entityId: string;
 }) {
   return new HostEntity({
     'source_data_stream.type': dataStreamType,
-    'source_data_stream.dataset': dataStreamDataset,
     'agent.name': agentName,
     'host.name': hostName,
+    'entity.displayName': hostName,
     'entity.id': entityId,
   });
 }

packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts

@@ -15,7 +15,8 @@ class ServiceEntity extends Serializable<EntityFields> {
     super({
       ...fields,
       'entity.type': 'service',
-      'entity.definitionId': 'latest',
+      'entity.definitionId': 'builtin_services_from_ecs_data',
+      'entity.identityFields': ['service.name'],
     });
   }
 }
@@ -35,6 +36,7 @@ export function serviceEntity({
 }) {
   return new ServiceEntity({
     'service.name': serviceName,
+    'entity.displayName': serviceName,
     'service.environment': environment,
     'source_data_stream.type': dataStreamType,
     'agent.name': agentName,

packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts

@@ -6,8 +6,6 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
-import { randomInt } from 'crypto';
 import { Fields } from '../entity';
 import { Serializable } from '../serializable';
 
@@ -180,3 +178,12 @@ export const log = {
   create,
   createMinimal,
 };
+
+function randomInt(min: number, max: number) {
+  if (min > max) {
+    throw new Error('Min value must be less than or equal to max value.');
+  }
+
+  const random = Math.floor(Math.random() * (max - min + 1)) + min;
+  return random;
+}

packages/kbn-apm-synthtrace/index.ts

@@ -16,6 +16,7 @@ export { InfraSynthtraceKibanaClient } from './src/lib/infra/infra_synthtrace_ki
 export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';
 export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
 export { EntitiesSynthtraceEsClient } from './src/lib/entities/entities_synthtrace_es_client';
+export { EntitiesSynthtraceKibanaClient } from './src/lib/entities/entities_synthtrace_kibana_client';
 export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
 export { OtelSynthtraceEsClient } from './src/lib/otel/otel_synthtrace_es_client';
 export {

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -19,7 +19,7 @@ import {
 import { Logger } from '../lib/utils/create_logger';
 import { ScenarioReturnType } from '../lib/utils/with_client';
 import { RunOptions } from './utils/parse_run_cli_flags';
-import { EntitiesSynthtraceKibanaClient } from '../lib/apm/client/entities_synthtrace_kibana_client';
+import { EntitiesSynthtraceKibanaClient } from '../lib/entities/entities_synthtrace_kibana_client';
 
 interface EsClients {
   apmEsClient: ApmSynthtraceEsClient;

packages/kbn-apm-synthtrace/src/cli/utils/get_entites_kibana_client.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { EntitiesSynthtraceKibanaClient } from '../../lib/apm/client/entities_synthtrace_kibana_client';
+import { EntitiesSynthtraceKibanaClient } from '../../lib/entities/entities_synthtrace_kibana_client';
 import { Logger } from '../../lib/utils/create_logger';
 
 export function getEntitiesKibanaClient({ target, logger }: { target: string; logger: Logger }) {

packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts

@@ -17,6 +17,10 @@ import { Logger } from '../utils/create_logger';
 
 export type EntitiesSynthtraceEsClientOptions = Omit<SynthtraceEsClientOptions, 'pipeline'>;
 
+interface Pipeline {
+  includeSerialization?: boolean;
+}
+
 export class EntitiesSynthtraceEsClient extends SynthtraceEsClient<EntityFields> {
   constructor(options: { client: Client; logger: Logger } & EntitiesSynthtraceEsClientOptions) {
     super({
@@ -25,13 +29,20 @@ export class EntitiesSynthtraceEsClient extends SynthtraceEsClient<EntityFields>
     });
     this.indices = ['.entities.v1.latest.builtin*'];
   }
+
+  getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
+    return entitiesPipeline({ includeSerialization });
+  }
 }
 
-function entitiesPipeline() {
+function entitiesPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
   return (base: Readable) => {
+    const serializationTransform = includeSerialization ? [getSerializeTransform()] : [];
+
     return pipeline(
+      // @ts-expect-error Some weird stuff here with the type definition for pipeline. We have tests!
       base,
-      getSerializeTransform(),
+      ...serializationTransform,
       lastSeenTimestampTransform(),
       getRoutingTransform(),
       getDedotTransform(),

packages/kbn-apm-synthtrace/src/lib/apm/client/entities_synthtrace_kibana_client.ts → packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_kibana_client.ts

@@ -8,9 +8,9 @@
  */
 
 import fetch from 'node-fetch';
-import { Logger } from '../../utils/create_logger';
-import { kibanaHeaders } from '../../shared/client_headers';
-import { getFetchAgent } from '../../../cli/utils/ssl';
+import { Logger } from '../utils/create_logger';
+import { kibanaHeaders } from '../shared/client_headers';
+import { getFetchAgent } from '../../cli/utils/ssl';
 
 interface EntityDefinitionResponse {
   definitions: Array<{ type: string; state: { installed: boolean; running: boolean } }>;

packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts

@@ -23,6 +23,10 @@ export const LogsCustom = 'logs@custom';
 
 export type LogsSynthtraceEsClientOptions = Omit<SynthtraceEsClientOptions, 'pipeline'>;
 
+interface Pipeline {
+  includeSerialization?: boolean;
+}
+
 export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
   constructor(options: { client: Client; logger: Logger } & LogsSynthtraceEsClientOptions) {
     super({
@@ -132,13 +136,22 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
       this.logger.error(`Custom pipeline creation failed: ${LogsCustom} - ${err.message}`);
     }
   }
+
+  getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
+    return logsPipeline({ includeSerialization });
+  }
 }
 
-function logsPipeline() {
+function logsPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
   return (base: Readable) => {
+    const serializationTransform = includeSerialization
+      ? [getSerializeTransform<LogDocument>()]
+      : [];
+
     return pipeline(
+      // @ts-expect-error Some weird stuff here with the type definition for pipeline. We have tests!
       base,
-      getSerializeTransform<LogDocument>(),
+      ...serializationTransform,
       getRoutingTransform('logs'),
       (err: unknown) => {
         if (err) {

tsconfig.base.json

@@ -1062,6 +1062,8 @@
       "@kbn/interactive-setup-test-endpoints-plugin/*": ["test/interactive_setup_api_integration/plugins/test_endpoints/*"],
       "@kbn/interpreter": ["packages/kbn-interpreter"],
       "@kbn/interpreter/*": ["packages/kbn-interpreter/*"],
+      "@kbn/inventory-e2e": ["x-pack/plugins/observability_solution/inventory/e2e"],
+      "@kbn/inventory-e2e/*": ["x-pack/plugins/observability_solution/inventory/e2e/*"],
       "@kbn/inventory-plugin": ["x-pack/plugins/observability_solution/inventory"],
       "@kbn/inventory-plugin/*": ["x-pack/plugins/observability_solution/inventory/*"],
       "@kbn/investigate-app-plugin": ["x-pack/plugins/observability_solution/investigate_app"],