[Logs Explorer][Discover] Move Logs Overview into Discover codebase (#…

…180262)

## 📓 Summary

Closes #180024 

This work aims to move the "Logs Overview" doc view created from the
logs-explorer app into the unified-doc-viewer plugin, creating and
registering this as a preset along the table and source doc views.

To keep control of whether this doc view should be displayed or not, an
`enabled` configuration flag is supported for every doc view and will be
used to determine whether a doc view should load or not in the view.
This `enabled` flag can be programmatically enabled/disabled by
`docView.id` using the 2 new methods added to the `DocViewsRegistry`,
the `enableById` and `disableById` ones.
The customization extension point does not register the content of the
tab, but is limited to enable/disable a preset overview tab.

To allow this change, some shared utilities between the flyout logic and
the smart fields feature have been copied into the `@kbn/discover-utils`
package. The utils will still live in the logs_explorer plugin and are
used by the smart fields feature until this is migrated too into
Discover.

## 💡 Reviewer hints

Although it seems a large PR, most of the changes are on moved files
from logs-explorer into unified-doc-viewer, which is logic already
reviewed. With these changes, there will be a follow-up PR to optimize
the shared parts with the other doc views.

## 🚶 Next steps

To keep the scope of this PR the smallest possible, here are some
changes I'll work out in upcoming PRs built on top of this one.

- [x] Implement a discover registry to enable registering external
features, such as the ObservabilityAIAssistant.
- [x] Integrate ObsAIAssistant with this work through the new discover
features registry.
- [x] Refactor the doc views to share duplicated logic.
- [x] Port the existing e2e tests for the logs overview tab into the
unified-doc-viewer plugin.

tonyghiani#1

---------

Co-authored-by: Marco Antonio Ghiani <[email protected]>
Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Davis McPhee <[email protected]>

.github/CODEOWNERS

@@ -1138,6 +1138,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /x-pack/test/functional/apps/dataset_quality @elastic/obs-ux-logs-team
 /x-pack/test_serverless/functional/test_suites/observability/dataset_quality @elastic/obs-ux-logs-team
 /x-pack/test_serverless/functional/test_suites/observability/ @elastic/obs-ux-logs-team
+/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview @elastic/obs-ux-logs-team
 
 # Observability onboarding tour
 /x-pack/plugins/observability_solution/observability_shared/public/components/tour @elastic/platform-onboarding

packages/kbn-discover-utils/index.ts

@@ -31,12 +31,17 @@ export {
   IgnoredReason,
   buildDataTableRecord,
   buildDataTableRecordList,
+  fieldConstants,
   formatFieldValue,
   formatHit,
   getDocId,
+  getLogDocumentOverview,
   getIgnoredReason,
+  getMessageFieldWithFallbacks,
   getShouldShowFieldHandler,
   isNestedFieldParent,
   isLegacyTableEnabled,
   usePager,
 } from './src';
+
+export * from './src/types';

packages/kbn-discover-utils/src/field_constants.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// Fields constants
+export const TIMESTAMP_FIELD = '@timestamp';
+export const HOST_NAME_FIELD = 'host.name';
+export const LOG_LEVEL_FIELD = 'log.level';
+export const MESSAGE_FIELD = 'message';
+export const ERROR_MESSAGE_FIELD = 'error.message';
+export const EVENT_ORIGINAL_FIELD = 'event.original';
+export const TRACE_ID_FIELD = 'trace.id';
+
+export const LOG_FILE_PATH_FIELD = 'log.file.path';
+export const DATASTREAM_NAMESPACE_FIELD = 'data_stream.namespace';
+export const DATASTREAM_DATASET_FIELD = 'data_stream.dataset';
+
+// Resource Fields
+export const AGENT_NAME_FIELD = 'agent.name';
+export const CLOUD_PROVIDER_FIELD = 'cloud.provider';
+export const CLOUD_REGION_FIELD = 'cloud.region';
+export const CLOUD_AVAILABILITY_ZONE_FIELD = 'cloud.availability_zone';
+export const CLOUD_PROJECT_ID_FIELD = 'cloud.project.id';
+export const CLOUD_INSTANCE_ID_FIELD = 'cloud.instance.id';
+export const SERVICE_NAME_FIELD = 'service.name';
+export const ORCHESTRATOR_CLUSTER_NAME_FIELD = 'orchestrator.cluster.name';
+export const ORCHESTRATOR_RESOURCE_ID_FIELD = 'orchestrator.resource.id';
+export const ORCHESTRATOR_NAMESPACE_FIELD = 'orchestrator.namespace';
+export const CONTAINER_NAME_FIELD = 'container.name';
+export const CONTAINER_ID_FIELD = 'container.id';
+
+// Degraded Docs
+export const DEGRADED_DOCS_FIELD = 'ignored_field_values';
+
+// Error Stacktrace
+export const ERROR_STACK_TRACE = 'error.stack_trace';
+export const ERROR_EXCEPTION_STACKTRACE = 'error.exception.stacktrace';
+export const ERROR_LOG_STACKTRACE = 'error.log.stacktrace';

packages/kbn-discover-utils/src/index.ts

@@ -7,5 +7,6 @@
  */
 
 export * from './constants';
+export * as fieldConstants from './field_constants';
 export * from './hooks';
 export * from './utils';

packages/kbn-discover-utils/src/types.ts

@@ -46,3 +46,44 @@ type FormattedHitPair = readonly [
  * Pairs array for each field in the hit
  */
 export type FormattedHit = FormattedHitPair[];
+
+export interface LogDocumentOverview
+  extends LogResourceFields,
+    LogStackTraceFields,
+    LogCloudFields {
+  '@timestamp': string;
+  'log.level'?: string;
+  message?: string;
+  'error.message'?: string;
+  'event.original'?: string;
+  'trace.id'?: string;
+  'log.file.path'?: string;
+  'data_stream.namespace': string;
+  'data_stream.dataset': string;
+}
+
+export interface LogResourceFields {
+  'host.name'?: string;
+  'service.name'?: string;
+  'agent.name'?: string;
+  'orchestrator.cluster.name'?: string;
+  'orchestrator.cluster.id'?: string;
+  'orchestrator.resource.id'?: string;
+  'orchestrator.namespace'?: string;
+  'container.name'?: string;
+  'container.id'?: string;
+}
+
+export interface LogStackTraceFields {
+  'error.stack_trace'?: string;
+  'error.exception.stacktrace'?: string;
+  'error.log.stacktrace'?: string;
+}
+
+export interface LogCloudFields {
+  'cloud.provider'?: string;
+  'cloud.region'?: string;
+  'cloud.availability_zone'?: string;
+  'cloud.project.id'?: string;
+  'cloud.instance.id'?: string;
+}

packages/kbn-discover-utils/src/utils/get_log_document_overview.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import { DataTableRecord, LogDocumentOverview, fieldConstants, formatFieldValue } from '../..';
+
+export function getLogDocumentOverview(
+  doc: DataTableRecord,
+  { dataView, fieldFormats }: { dataView: DataView; fieldFormats: FieldFormatsStart }
+): LogDocumentOverview {
+  const formatField = <T extends keyof LogDocumentOverview>(field: T) => {
+    return (
+      field in doc.flattened
+        ? formatFieldValue(
+            doc.flattened[field],
+            doc.raw,
+            fieldFormats,
+            dataView,
+            dataView.fields.getByName(field)
+          )
+        : undefined
+    ) as LogDocumentOverview[T];
+  };
+
+  const levelArray = doc.flattened[fieldConstants.LOG_LEVEL_FIELD];
+  const level =
+    Array.isArray(levelArray) && levelArray.length ? levelArray[0].toLowerCase() : levelArray;
+  const messageArray = doc.flattened[fieldConstants.MESSAGE_FIELD];
+  const message =
+    Array.isArray(messageArray) && messageArray.length ? messageArray[0] : messageArray;
+  const errorMessageArray = doc.flattened[fieldConstants.ERROR_MESSAGE_FIELD];
+  const errorMessage =
+    Array.isArray(errorMessageArray) && errorMessageArray.length
+      ? errorMessageArray[0]
+      : errorMessageArray;
+  const eventOriginalArray = doc.flattened[fieldConstants.EVENT_ORIGINAL_FIELD];
+  const eventOriginal =
+    Array.isArray(eventOriginalArray) && eventOriginalArray.length
+      ? eventOriginalArray[0]
+      : eventOriginalArray;
+  const timestamp = formatField(fieldConstants.TIMESTAMP_FIELD);
+
+  // Service
+  const serviceName = formatField(fieldConstants.SERVICE_NAME_FIELD);
+  const traceId = formatField(fieldConstants.TRACE_ID_FIELD);
+
+  // Infrastructure
+  const hostname = formatField(fieldConstants.HOST_NAME_FIELD);
+  const orchestratorClusterName = formatField(fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD);
+  const orchestratorResourceId = formatField(fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD);
+
+  // Cloud
+  const cloudProvider = formatField(fieldConstants.CLOUD_PROVIDER_FIELD);
+  const cloudRegion = formatField(fieldConstants.CLOUD_REGION_FIELD);
+  const cloudAz = formatField(fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD);
+  const cloudProjectId = formatField(fieldConstants.CLOUD_PROJECT_ID_FIELD);
+  const cloudInstanceId = formatField(fieldConstants.CLOUD_INSTANCE_ID_FIELD);
+
+  // Other
+  const logFilePath = formatField(fieldConstants.LOG_FILE_PATH_FIELD);
+  const namespace = formatField(fieldConstants.DATASTREAM_NAMESPACE_FIELD);
+  const dataset = formatField(fieldConstants.DATASTREAM_DATASET_FIELD);
+  const agentName = formatField(fieldConstants.AGENT_NAME_FIELD);
+
+  return {
+    [fieldConstants.LOG_LEVEL_FIELD]: level,
+    [fieldConstants.TIMESTAMP_FIELD]: timestamp,
+    [fieldConstants.MESSAGE_FIELD]: message,
+    [fieldConstants.ERROR_MESSAGE_FIELD]: errorMessage,
+    [fieldConstants.EVENT_ORIGINAL_FIELD]: eventOriginal,
+    [fieldConstants.SERVICE_NAME_FIELD]: serviceName,
+    [fieldConstants.TRACE_ID_FIELD]: traceId,
+    [fieldConstants.HOST_NAME_FIELD]: hostname,
+    [fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD]: orchestratorClusterName,
+    [fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD]: orchestratorResourceId,
+    [fieldConstants.CLOUD_PROVIDER_FIELD]: cloudProvider,
+    [fieldConstants.CLOUD_REGION_FIELD]: cloudRegion,
+    [fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD]: cloudAz,
+    [fieldConstants.CLOUD_PROJECT_ID_FIELD]: cloudProjectId,
+    [fieldConstants.CLOUD_INSTANCE_ID_FIELD]: cloudInstanceId,
+    [fieldConstants.LOG_FILE_PATH_FIELD]: logFilePath,
+    [fieldConstants.DATASTREAM_NAMESPACE_FIELD]: namespace,
+    [fieldConstants.DATASTREAM_DATASET_FIELD]: dataset,
+    [fieldConstants.AGENT_NAME_FIELD]: agentName,
+  };
+}

packages/kbn-discover-utils/src/utils/get_message_field_with_fallbacks.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { fieldConstants } from '..';
+import { LogDocumentOverview } from '../types';
+
+export const getMessageFieldWithFallbacks = (doc: LogDocumentOverview) => {
+  const rankingOrder = [
+    fieldConstants.MESSAGE_FIELD,
+    fieldConstants.ERROR_MESSAGE_FIELD,
+    fieldConstants.EVENT_ORIGINAL_FIELD,
+  ] as const;
+
+  for (const rank of rankingOrder) {
+    if (doc[rank] !== undefined && doc[rank] !== null) {
+      return { field: rank, value: doc[rank] };
+    }
+  }
+
+  // If none of the ranks (fallbacks) are present
+  return { field: undefined };
+};

packages/kbn-discover-utils/src/utils/index.ts

@@ -11,6 +11,8 @@ export * from './format_hit';
 export * from './format_value';
 export * from './get_doc_id';
 export * from './get_ignored_reason';
+export * from './get_log_document_overview';
+export * from './get_message_field_with_fallbacks';
 export * from './get_should_show_field_handler';
 export * from './nested_fields';
 export { isLegacyTableEnabled } from './is_legacy_table_enabled';

packages/kbn-unified-doc-viewer/src/components/doc_viewer/doc_viewer.tsx

@@ -22,22 +22,24 @@ export interface DocViewerProps extends DocViewRenderProps {
  * a `render` function.
  */
 export function DocViewer({ docViews, ...renderProps }: DocViewerProps) {
-  const tabs = docViews.map(({ id, title, render, component }: DocView) => {
-    return {
-      id: `kbn_doc_viewer_tab_${id}`,
-      name: title,
-      content: (
-        <DocViewerTab
-          id={id}
-          title={title}
-          component={component}
-          renderProps={renderProps}
-          render={render}
-        />
-      ),
-      ['data-test-subj']: `docViewerTab-${id}`,
-    };
-  });
+  const tabs = docViews
+    .filter(({ enabled }) => enabled) // Filter out disabled doc views
+    .map(({ id, title, render, component }: DocView) => {
+      return {
+        id: `kbn_doc_viewer_tab_${id}`,
+        name: title,
+        content: (
+          <DocViewerTab
+            id={id}
+            title={title}
+            component={component}
+            renderProps={renderProps}
+            render={render}
+          />
+        ),
+        ['data-test-subj']: `docViewerTab-${id}`,
+      };
+    });
 
   if (!tabs.length) {
     // There's a minimum of 2 tabs active in Discover.

packages/kbn-unified-doc-viewer/src/services/doc_views_registry.test.tsx

@@ -74,6 +74,25 @@ describe('DocViewerRegistry', () => {
     });
   });
 
+  describe('#enableById & #disableById', () => {
+    test('should enable/disable a doc view given the passed id', () => {
+      const registry = new DocViewsRegistry([fnDocView, componentDocView]);
+
+      const docViews = registry.getAll();
+
+      expect(docViews[0]).toHaveProperty('enabled', true);
+      expect(docViews[1]).toHaveProperty('enabled', true);
+
+      registry.disableById('function-doc-view');
+
+      expect(registry.getAll()[0]).toHaveProperty('enabled', false);
+
+      registry.enableById('function-doc-view');
+
+      expect(registry.getAll()[0]).toHaveProperty('enabled', true);
+    });
+  });
+
   describe('#clone', () => {
     test('should return a new DocViewRegistry instance starting from the current one', () => {
       const registry = new DocViewsRegistry([fnDocView, componentDocView]);
@@ -84,6 +103,24 @@ describe('DocViewerRegistry', () => {
       expect(docViews[0]).toHaveProperty('id', 'function-doc-view');
       expect(docViews[1]).toHaveProperty('id', 'component-doc-view');
       expect(registry).not.toBe(clonedRegistry);
+
+      // Test against shared references between clones
+      expect(clonedRegistry).not.toBe(registry);
+
+      // Mutating a cloned registry should not affect the original registry
+      registry.disableById('function-doc-view');
+      expect(registry.getAll()[0]).toHaveProperty('enabled', false);
+      expect(clonedRegistry.getAll()[0]).toHaveProperty('enabled', true);
+
+      clonedRegistry.add({
+        id: 'additional-doc-view',
+        order: 20,
+        title: 'Render function',
+        render: jest.fn(),
+      });
+
+      expect(registry.getAll().length).toBe(2);
+      expect(clonedRegistry.getAll().length).toBe(3);
     });
   });
 });