[Logs+] Implement Logs Dataset selector

.github/CODEOWNERS

@@ -328,6 +328,7 @@ packages/kbn-dev-utils @elastic/kibana-operations
 examples/developer_examples @elastic/appex-sharedux
 examples/discover_customization_examples @elastic/kibana-data-discovery
 x-pack/plugins/discover_enhanced @elastic/kibana-data-discovery
+x-pack/plugins/discover_log_explorer @elastic/infra-monitoring-ui
 src/plugins/discover @elastic/kibana-data-discovery
 packages/kbn-doc-links @elastic/docs
 packages/kbn-docs-utils @elastic/kibana-operations

docs/developer/plugin-list.asciidoc

@@ -516,6 +516,10 @@ Plugin server-side only. Plugin has three main functions:
 |Contains the enhancements to the OSS discover app.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/discover_log_explorer/README.md[discoverLogExplorer]
+|This plugin registers a log-explorer profile using the Discover customization framework, offering several affordances specifically designed for log consumption.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/ecs_data_quality_dashboard/README.md[ecsDataQualityDashboard]
 |This plugin implements (server) APIs used to render the content of the Data Quality dashboard.
 

package.json

@@ -370,6 +370,7 @@
     "@kbn/developer-examples-plugin": "link:examples/developer_examples",
     "@kbn/discover-customization-examples-plugin": "link:examples/discover_customization_examples",
     "@kbn/discover-enhanced-plugin": "link:x-pack/plugins/discover_enhanced",
+    "@kbn/discover-log-explorer-plugin": "link:x-pack/plugins/discover_log_explorer",
     "@kbn/discover-plugin": "link:src/plugins/discover",
     "@kbn/doc-links": "link:packages/kbn-doc-links",
     "@kbn/dom-drag-drop": "link:packages/kbn-dom-drag-drop",

packages/kbn-io-ts-utils/index.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-export type { IndexPatternType } from './src/index_pattern_rt';
+export type { IndexPattern } from './src/index_pattern_rt';
 export type { NonEmptyString, NonEmptyStringBrand } from './src/non_empty_string_rt';
 
 export { deepExactRt } from './src/deep_exact_rt';

packages/kbn-io-ts-utils/src/index_pattern_rt/index.ts

@@ -25,12 +25,10 @@ export interface IndexPatternBrand {
   readonly IndexPattern: unique symbol;
 }
 
-type IndexPattern = rt.Branded<string, IndexPatternBrand>;
+export type IndexPattern = rt.Branded<string, IndexPatternBrand>;
 
 export const indexPatternRt = rt.brand(
   rt.string,
   (pattern): pattern is IndexPattern => validateIndexPattern(pattern),
   'IndexPattern'
 );
-
-export type IndexPatternType = rt.TypeOf<typeof indexPatternRt>;

packages/kbn-optimizer/limits.yml

@@ -36,6 +36,7 @@ pageLoadAssetSize:
   devTools: 38637
   discover: 99999
   discoverEnhanced: 42730
+  discoverLogExplorer: 39045
   embeddable: 87309
   embeddableEnhanced: 22107
   enterpriseSearch: 50858

src/dev/storybook/aliases.ts

@@ -25,6 +25,7 @@ export const storybookAliases = {
   dashboard: 'src/plugins/dashboard/.storybook',
   data: 'src/plugins/data/.storybook',
   discover: 'src/plugins/discover/.storybook',
+  discover_log_explorer: 'x-pack/plugins/discover_log_explorer/.storybook',
   embeddable: 'src/plugins/embeddable/.storybook',
   es_ui_shared: 'src/plugins/es_ui_shared/.storybook',
   expression_error: 'src/plugins/expression_error/.storybook',

src/plugins/discover/public/application/main/services/discover_state.test.ts

@@ -646,7 +646,7 @@ describe('Test discover state actions', () => {
     const { state } = await getState('/', savedSearchMock);
     await state.actions.loadSavedSearch({ savedSearchId: savedSearchMock.id });
     const unsubscribe = state.actions.initializeAndSync();
-    await state.actions.onCreateDefaultAdHocDataView('ad-hoc-test');
+    await state.actions.onCreateDefaultAdHocDataView({ title: 'ad-hoc-test' });
     expect(state.appState.getState().index).toBe('ad-hoc-id');
     expect(state.internalState.getState().adHocDataViews[0].id).toBe('ad-hoc-id');
     unsubscribe();

src/plugins/discover/public/application/main/services/discover_state.ts

@@ -135,7 +135,7 @@ export interface DiscoverStateContainer {
      * Used by the Data View Picker
      * @param pattern
      */
-    onCreateDefaultAdHocDataView: (pattern: string) => Promise<void>;
+    onCreateDefaultAdHocDataView: (dataViewSpec: DataViewSpec) => Promise<void>;
     /**
      * Triggered when a new data view is created
      * @param dataView
@@ -385,10 +385,8 @@ export function getDiscoverStateContainer({
     };
   };
 
-  const onCreateDefaultAdHocDataView = async (pattern: string) => {
-    const newDataView = await services.dataViews.create({
-      title: pattern,
-    });
+  const onCreateDefaultAdHocDataView = async (dataViewSpec: DataViewSpec) => {
+    const newDataView = await services.dataViews.create(dataViewSpec);
     if (newDataView.fields.getByName('@timestamp')?.type === 'date') {
       newDataView.timeFieldName = '@timestamp';
     }

src/plugins/discover/public/index.ts

@@ -15,5 +15,7 @@ export function plugin(initializerContext: PluginInitializerContext) {
 }
 
 export type { ISearchEmbeddable, SearchInput } from './embeddable';
+export type { DiscoverStateContainer } from './application/main/services/discover_state';
+export type { CustomizationCallback } from './customizations';
 export { SEARCH_EMBEDDABLE_TYPE, SEARCH_EMBEDDABLE_CELL_ACTIONS_TRIGGER_ID } from './embeddable';
 export { loadSharingDataHelpers } from './utils';

src/plugins/unified_search/public/dataview_picker/data_view_picker.tsx

@@ -8,7 +8,7 @@
 
 import React from 'react';
 import type { EuiButtonProps, EuiSelectableProps } from '@elastic/eui';
-import type { DataView, DataViewListItem } from '@kbn/data-views-plugin/public';
+import type { DataView, DataViewListItem, DataViewSpec } from '@kbn/data-views-plugin/public';
 import type { AggregateQuery, Query } from '@kbn/es-query';
 import { ChangeDataView } from './change_dataview';
 
@@ -73,7 +73,7 @@ export interface DataViewPickerProps {
    */
   onDataViewCreated?: () => void;
 
-  onCreateDefaultAdHocDataView?: (pattern: string) => void;
+  onCreateDefaultAdHocDataView?: (dataViewSpec: DataViewSpec) => void;
   /**
    * List of the supported text based languages (SQL, ESQL) etc.
    * Defined per application, if not provided, no text based languages

src/plugins/unified_search/public/dataview_picker/data_view_selector.tsx

@@ -8,7 +8,7 @@
 
 import React, { Fragment, useEffect, useRef, useState } from 'react';
 import type { EuiSelectableProps } from '@elastic/eui';
-import type { DataViewListItem } from '@kbn/data-views-plugin/public';
+import type { DataViewListItem, DataViewSpec } from '@kbn/data-views-plugin/public';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { DataViewsList } from './dataview_list';
 import { IUnifiedSearchPluginServices } from '../types';
@@ -22,7 +22,7 @@ export interface DataViewSelectorProps {
   isTextBasedLangSelected: boolean;
   setPopoverIsOpen: (isOpen: boolean) => void;
   onChangeDataView: (dataViewId: string) => void;
-  onCreateDefaultAdHocDataView?: (pattern: string) => void;
+  onCreateDefaultAdHocDataView?: (dataViewSpec: DataViewSpec) => void;
 }
 
 export const DataViewSelector = ({

src/plugins/unified_search/public/dataview_picker/explore_matching_button.tsx

@@ -10,12 +10,13 @@ import React from 'react';
 import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSpacer, useEuiTheme } from '@elastic/eui';
 import { css } from '@emotion/react';
 import { i18n } from '@kbn/i18n';
+import { DataViewSpec } from '@kbn/data-views-plugin/common';
 
 interface ExploreMatchingButtonProps {
   noDataViewMatches: boolean;
   indexMatches: number;
   dataViewSearchString: string;
-  onCreateDefaultAdHocDataView?: (pattern: string) => void;
+  onCreateDefaultAdHocDataView?: (dataViewSpec: DataViewSpec) => void;
   setPopoverIsOpen: (isOpen: boolean) => void;
 }
 
@@ -47,7 +48,7 @@ export const ExploreMatchingButton = ({
             data-test-subj="explore-matching-indices-button"
             onClick={() => {
               setPopoverIsOpen(false);
-              onCreateDefaultAdHocDataView(dataViewSearchString);
+              onCreateDefaultAdHocDataView({ title: dataViewSearchString });
             }}
           >
             {i18n.translate('unifiedSearch.query.queryBar.indexPattern.createForMatchingIndices', {

tsconfig.base.json

@@ -650,6 +650,8 @@
       "@kbn/discover-customization-examples-plugin/*": ["examples/discover_customization_examples/*"],
       "@kbn/discover-enhanced-plugin": ["x-pack/plugins/discover_enhanced"],
       "@kbn/discover-enhanced-plugin/*": ["x-pack/plugins/discover_enhanced/*"],
+      "@kbn/discover-log-explorer-plugin": ["x-pack/plugins/discover_log_explorer"],
+      "@kbn/discover-log-explorer-plugin/*": ["x-pack/plugins/discover_log_explorer/*"],
       "@kbn/discover-plugin": ["src/plugins/discover"],
       "@kbn/discover-plugin/*": ["src/plugins/discover/*"],
       "@kbn/doc-links": ["packages/kbn-doc-links"],

x-pack/.i18nrc.json

@@ -20,6 +20,7 @@
     "xpack.customBranding": "plugins/custom_branding",
     "xpack.dashboard": "plugins/dashboard_enhanced",
     "xpack.discover": "plugins/discover_enhanced",
+    "xpack.discoverLogExplorer": "plugins/discover_log_explorer",
     "xpack.crossClusterReplication": "plugins/cross_cluster_replication",
     "xpack.elasticAssistant": "packages/kbn-elastic-assistant",
     "xpack.embeddableEnhanced": "plugins/embeddable_enhanced",

x-pack/plugins/discover_log_explorer/.storybook/__mocks__/package_icon.tsx

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React from 'react';
+import { EuiIcon } from '@elastic/eui';
+
+// Export mock package icon that doesn't trigger http requests
+export const PackageIcon = () => <EuiIcon type="package" style={{ marginRight: 8 }} />;

x-pack/plugins/discover_log_explorer/.storybook/main.js

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const defaultConfig = require('@kbn/storybook').defaultConfig;
+
+module.exports = {
+  ...defaultConfig,
+  stories: ['../**/*.stories.mdx', ...defaultConfig.stories],
+  webpackFinal: async (config) => {
+    const originalConfig = await defaultConfig.webpackFinal(config);
+
+    // Mock fleet plugin for PackageIcon component
+    originalConfig.resolve.alias['@kbn/fleet-plugin/public'] = require.resolve(
+      './__mocks__/package_icon'
+    );
+    return originalConfig;
+  },
+};

x-pack/plugins/discover_log_explorer/.storybook/preview.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+export const parameters = {
+  docs: {
+    source: {
+      type: 'code', // without this, stories in mdx documents freeze the browser
+    },
+  },
+};

x-pack/plugins/discover_log_explorer/README.md

@@ -0,0 +1,8 @@
+# Discover Log Explorer
+
+This plugin registers a `log-explorer` profile using the Discover customization framework, offering several affordances specifically designed for log consumption.
+
+The plugin enhances the capabilities of Discover in the following ways:
+
+- **Dataset selector**: this customization on the Discover page replaces the DataViews picker with a Logs dataset selector built ad-hoc to provide a better experience when navigating throught all the available datasets.
+

x-pack/plugins/discover_log_explorer/common/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const LOG_EXPLORER_PROFILE_ID = 'log-explorer';

x-pack/plugins/discover_log_explorer/common/datasets/errors.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable max-classes-per-file */
+
+export class FindIntegrationsError extends Error {
+  constructor(message: string) {
+    super(message);
+    Object.setPrototypeOf(this, new.target.prototype);
+    this.name = 'FindIntegrationsError';
+  }
+}
+
+export class FindDatasetsError extends Error {
+  constructor(message: string) {
+    super(message);
+    Object.setPrototypeOf(this, new.target.prototype);
+    this.name = 'FindDatasetsError';
+  }
+}

x-pack/plugins/discover_log_explorer/common/datasets/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './types';
+export { Dataset } from './models/dataset';
+export { Integration } from './models/integration';

x-pack/plugins/discover_log_explorer/common/datasets/models/dataset.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DataViewSpec } from '@kbn/data-views-plugin/common';
+import { DatasetId, DatasetType, IntegrationType } from '../types';
+
+type IntegrationBase = Pick<IntegrationType, 'name' | 'version'>;
+
+export class Dataset {
+  id: DatasetId;
+  name: DatasetType['name'];
+  title: DatasetType['title'];
+  parentIntegration?: IntegrationBase;
+
+  private constructor(dataset: DatasetType, parentIntegration?: IntegrationType) {
+    this.id = `dataset-${dataset.name}` as DatasetId;
+    this.name = dataset.name;
+    this.title = dataset.title ?? dataset.name;
+    this.parentIntegration = parentIntegration && {
+      name: parentIntegration.name,
+      version: parentIntegration.version,
+    };
+  }
+
+  toDataviewSpec(): DataViewSpec {
+    // Invert the property because the API returns the index pattern as `name` and a readable name as `title`
+    return {
+      id: this.id,
+      name: this.title,
+      title: this.name,
+    };
+  }
+
+  public static create(dataset: DatasetType, parentIntegration?: IntegrationType) {
+    return new Dataset(dataset, parentIntegration);
+  }
+}

x-pack/plugins/discover_log_explorer/common/datasets/models/integration.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Dataset } from './dataset';
+import { IntegrationId, IntegrationType } from '../types';
+
+export class Integration {
+  id: IntegrationId;
+  name: IntegrationType['name'];
+  status: IntegrationType['status'];
+  version: IntegrationType['version'];
+  datasets: Dataset[];
+
+  private constructor(integration: Integration) {
+    this.id = integration.id;
+    this.name = integration.name;
+    this.status = integration.status;
+    this.version = integration.version;
+    this.datasets = integration.datasets;
+  }
+
+  public static create(integration: IntegrationType) {
+    return new Integration({
+      ...integration,
+      id: `integration-${integration.name}-${integration.version}` as IntegrationId,
+      datasets: integration.dataStreams.map((dataset) => Dataset.create(dataset, integration)),
+    });
+  }
+}