[Logs+] Implement Logs Dataset selector

.github/CODEOWNERS

@@ -327,6 +327,7 @@ packages/kbn-dev-utils @elastic/kibana-operations
 examples/developer_examples @elastic/appex-sharedux
 examples/discover_customization_examples @elastic/kibana-data-discovery
 x-pack/plugins/discover_enhanced @elastic/kibana-data-discovery
+x-pack/plugins/discover_log_explorer @elastic/infra-monitoring-ui
 src/plugins/discover @elastic/kibana-data-discovery
 packages/kbn-doc-links @elastic/kibana-docs
 packages/kbn-docs-utils @elastic/kibana-operations

docs/developer/plugin-list.asciidoc

@@ -520,6 +520,10 @@ Plugin server-side only. Plugin has three main functions:
 |Contains the enhancements to the OSS discover app.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/discover_log_explorer/README.md[discoverLogExplorer]
+|This plugin exposes and registers Logs+ features.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/ecs_data_quality_dashboard/README.md[ecsDataQualityDashboard]
 |This plugin implements (server) APIs used to render the content of the Data Quality dashboard.
 

package.json

@@ -369,6 +369,7 @@
     "@kbn/developer-examples-plugin": "link:examples/developer_examples",
     "@kbn/discover-customization-examples-plugin": "link:examples/discover_customization_examples",
     "@kbn/discover-enhanced-plugin": "link:x-pack/plugins/discover_enhanced",
+    "@kbn/discover-log-explorer-plugin": "link:x-pack/plugins/discover_log_explorer",
     "@kbn/discover-plugin": "link:src/plugins/discover",
     "@kbn/doc-links": "link:packages/kbn-doc-links",
     "@kbn/dom-drag-drop": "link:packages/kbn-dom-drag-drop",

packages/kbn-optimizer/limits.yml

@@ -36,6 +36,7 @@ pageLoadAssetSize:
   devTools: 38637
   discover: 99999
   discoverEnhanced: 42730
+  discoverLogExplorer: 39045
   embeddable: 87309
   embeddableEnhanced: 22107
   enterpriseSearch: 50858

src/dev/storybook/aliases.ts

@@ -25,6 +25,7 @@ export const storybookAliases = {
   dashboard: 'src/plugins/dashboard/.storybook',
   data: 'src/plugins/data/.storybook',
   discover: 'src/plugins/discover/.storybook',
+  discover_log_explorer: 'x-pack/plugins/discover_log_explorer/.storybook',
   embeddable: 'src/plugins/embeddable/.storybook',
   es_ui_shared: 'src/plugins/es_ui_shared/.storybook',
   expression_error: 'src/plugins/expression_error/.storybook',

src/plugins/discover/public/application/main/services/discover_state.test.ts

@@ -646,7 +646,7 @@ describe('Test discover state actions', () => {
     const { state } = await getState('/', savedSearchMock);
     await state.actions.loadSavedSearch({ savedSearchId: savedSearchMock.id });
     const unsubscribe = state.actions.initializeAndSync();
-    await state.actions.onCreateDefaultAdHocDataView('ad-hoc-test');
+    await state.actions.onCreateDefaultAdHocDataView({ title: 'ad-hoc-test' });
     expect(state.appState.getState().index).toBe('ad-hoc-id');
     expect(state.internalState.getState().adHocDataViews[0].id).toBe('ad-hoc-id');
     unsubscribe();

src/plugins/discover/public/application/main/services/discover_state.ts

@@ -135,7 +135,7 @@ export interface DiscoverStateContainer {
      * Used by the Data View Picker
      * @param pattern
      */
-    onCreateDefaultAdHocDataView: (pattern: string) => Promise<void>;
+    onCreateDefaultAdHocDataView: (dataViewSpec: DataViewSpec) => Promise<void>;
     /**
      * Triggered when a new data view is created
      * @param dataView
@@ -388,10 +388,8 @@ export function getDiscoverStateContainer({
     };
   };
 
-  const onCreateDefaultAdHocDataView = async (pattern: string) => {
-    const newDataView = await services.dataViews.create({
-      title: pattern,
-    });
+  const onCreateDefaultAdHocDataView = async (dataViewSpec: DataViewSpec) => {
+    const newDataView = await services.dataViews.create(dataViewSpec);
     if (newDataView.fields.getByName('@timestamp')?.type === 'date') {
       newDataView.timeFieldName = '@timestamp';
     }

src/plugins/discover/public/index.ts

@@ -15,5 +15,6 @@ export function plugin(initializerContext: PluginInitializerContext) {
 }
 
 export type { ISearchEmbeddable, SearchInput } from './embeddable';
+export type { DiscoverStateContainer } from './application/main/services/discover_state';
 export { SEARCH_EMBEDDABLE_TYPE } from './embeddable';
 export { loadSharingDataHelpers } from './utils';

src/plugins/unified_search/public/dataview_picker/data_view_picker.tsx

@@ -8,7 +8,7 @@
 
 import React from 'react';
 import type { EuiButtonProps, EuiSelectableProps } from '@elastic/eui';
-import type { DataView, DataViewListItem } from '@kbn/data-views-plugin/public';
+import type { DataView, DataViewListItem, DataViewSpec } from '@kbn/data-views-plugin/public';
 import type { AggregateQuery, Query } from '@kbn/es-query';
 import { ChangeDataView } from './change_dataview';
 
@@ -73,7 +73,7 @@ export interface DataViewPickerProps {
    */
   onDataViewCreated?: () => void;
 
-  onCreateDefaultAdHocDataView?: (pattern: string) => void;
+  onCreateDefaultAdHocDataView?: (dataViewSpec: DataViewSpec) => void;
   /**
    * List of the supported text based languages (SQL, ESQL) etc.
    * Defined per application, if not provided, no text based languages

src/plugins/unified_search/public/dataview_picker/data_view_selector.tsx

@@ -8,7 +8,7 @@
 
 import React, { Fragment, useEffect, useRef, useState } from 'react';
 import type { EuiSelectableProps } from '@elastic/eui';
-import type { DataViewListItem } from '@kbn/data-views-plugin/public';
+import type { DataViewListItem, DataViewSpec } from '@kbn/data-views-plugin/public';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { DataViewsList } from './dataview_list';
 import { IUnifiedSearchPluginServices } from '../types';
@@ -22,7 +22,7 @@ export interface DataViewSelectorProps {
   isTextBasedLangSelected: boolean;
   setPopoverIsOpen: (isOpen: boolean) => void;
   onChangeDataView: (dataViewId: string) => void;
-  onCreateDefaultAdHocDataView?: (pattern: string) => void;
+  onCreateDefaultAdHocDataView?: (dataViewSpec: DataViewSpec) => void;
 }
 
 export const DataViewSelector = ({

src/plugins/unified_search/public/dataview_picker/explore_matching_button.tsx

@@ -10,12 +10,13 @@ import React from 'react';
 import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSpacer, useEuiTheme } from '@elastic/eui';
 import { css } from '@emotion/react';
 import { i18n } from '@kbn/i18n';
+import { DataViewSpec } from '@kbn/data-views-plugin/common';
 
 interface ExploreMatchingButtonProps {
   noDataViewMatches: boolean;
   indexMatches: number;
   dataViewSearchString: string;
-  onCreateDefaultAdHocDataView?: (pattern: string) => void;
+  onCreateDefaultAdHocDataView?: (dataViewSpec: DataViewSpec) => void;
   setPopoverIsOpen: (isOpen: boolean) => void;
 }
 
@@ -47,7 +48,7 @@ export const ExploreMatchingButton = ({
             data-test-subj="explore-matching-indices-button"
             onClick={() => {
               setPopoverIsOpen(false);
-              onCreateDefaultAdHocDataView(dataViewSearchString);
+              onCreateDefaultAdHocDataView({ title: dataViewSearchString });
             }}
           >
             {i18n.translate('unifiedSearch.query.queryBar.indexPattern.createForMatchingIndices', {

tsconfig.base.json

@@ -648,6 +648,8 @@
       "@kbn/discover-customization-examples-plugin/*": ["examples/discover_customization_examples/*"],
       "@kbn/discover-enhanced-plugin": ["x-pack/plugins/discover_enhanced"],
       "@kbn/discover-enhanced-plugin/*": ["x-pack/plugins/discover_enhanced/*"],
+      "@kbn/discover-log-explorer-plugin": ["x-pack/plugins/discover_log_explorer"],
+      "@kbn/discover-log-explorer-plugin/*": ["x-pack/plugins/discover_log_explorer/*"],
       "@kbn/discover-plugin": ["src/plugins/discover"],
       "@kbn/discover-plugin/*": ["src/plugins/discover/*"],
       "@kbn/doc-links": ["packages/kbn-doc-links"],

x-pack/.i18nrc.json

@@ -20,6 +20,7 @@
     "xpack.customBranding": "plugins/custom_branding",
     "xpack.dashboard": "plugins/dashboard_enhanced",
     "xpack.discover": "plugins/discover_enhanced",
+    "xpack.discoverLogExplorer": "plugins/discover_log_explorer",
     "xpack.crossClusterReplication": "plugins/cross_cluster_replication",
     "xpack.elasticAssistant": "packages/kbn-elastic-assistant",
     "xpack.embeddableEnhanced": "plugins/embeddable_enhanced",

x-pack/plugins/discover_log_explorer/.storybook/__mocks__/package_icon.tsx

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React from 'react';
+import { EuiIcon } from '@elastic/eui';
+
+// Export mock package icon that doesn't trigger http requests
+export const PackageIcon = () => <EuiIcon type="package" style={{ marginRight: 8 }} />;

x-pack/plugins/discover_log_explorer/.storybook/main.js

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const defaultConfig = require('@kbn/storybook').defaultConfig;
+
+module.exports = {
+  ...defaultConfig,
+  stories: ['../**/*.stories.mdx', ...defaultConfig.stories],
+  webpackFinal: async (config) => {
+    const originalConfig = await defaultConfig.webpackFinal(config);
+
+    // Mock fleet plugin for PackageIcon component
+    originalConfig.resolve.alias['@kbn/fleet-plugin/public'] = require.resolve(
+      './__mocks__/package_icon'
+    );
+    return originalConfig;
+  },
+};

x-pack/plugins/discover_log_explorer/.storybook/preview.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+export const parameters = {
+  docs: {
+    source: {
+      type: 'code', // without this, stories in mdx documents freeze the browser
+    },
+  },
+};

x-pack/plugins/discover_log_explorer/README.md

@@ -0,0 +1,3 @@
+# Discover Log Explorer
+
+This plugin exposes and registers Logs+ features.

x-pack/plugins/discover_log_explorer/common/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const DISCOVER_LOG_EXPLORER_PROFILE_ID = 'discover-log-explorer';

x-pack/plugins/discover_log_explorer/common/datasets/errors.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable max-classes-per-file */
+
+export class FindIntegrationsError extends Error {
+  constructor(message: string) {
+    super(message);
+    Object.setPrototypeOf(this, new.target.prototype);
+    this.name = 'FindIntegrationsError';
+  }
+}
+
+export class FindDatasetsError extends Error {
+  constructor(message: string) {
+    super(message);
+    Object.setPrototypeOf(this, new.target.prototype);
+    this.name = 'FindDatasetsError';
+  }
+}

x-pack/plugins/discover_log_explorer/common/datasets/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './types';

x-pack/plugins/discover_log_explorer/common/datasets/types.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { indexPatternRt } from '@kbn/io-ts-utils';
+import * as rt from 'io-ts';
+
+export const datasetRT = rt.exact(
+  rt.intersection([
+    rt.type({
+      name: indexPatternRt,
+    }),
+    rt.partial({
+      title: rt.string,
+    }),
+  ])
+);
+
+const integrationStatusRT = rt.union([
+  rt.literal('installed'),
+  rt.literal('installing'),
+  rt.literal('install_failed'),
+]);
+
+export const integrationRT = rt.type({
+  name: rt.string,
+  status: integrationStatusRT,
+  version: rt.string,
+  dataStreams: rt.array(datasetRT),
+});
+
+export type Dataset = rt.TypeOf<typeof datasetRT>;
+export type Integration = rt.TypeOf<typeof integrationRT>;

x-pack/plugins/discover_log_explorer/common/datasets/v1/common.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import * as rt from 'io-ts';
+import { isEmpty, mapValues, omitBy } from 'lodash';
+import { Integration } from '../types';
+import { FindDatasetsRequestQuery } from './find_datasets';
+import { FindIntegrationsRequestQuery } from './find_integrations';
+
+/**
+ * Constants
+ */
+export const DATASETS_URL = '/api/fleet/epm/datasets';
+export const INTEGRATIONS_URL = '/api/fleet/epm/packages/installed';
+
+/**
+ * Common types
+ */
+export const sortOrderRT = rt.union([rt.literal('asc'), rt.literal('desc')]);
+export type SortOrder = rt.TypeOf<typeof sortOrderRT>;
+export type IntegrationId = `integration-${string}-${string}`;
+
+/**
+ * Getters
+ */
+export const getIntegrationId = (integration: Integration): IntegrationId =>
+  `integration-${integration.name}-${integration.version}`;
+
+/**
+ * Utils
+ */
+function stringifyByProp(
+  obj: FindIntegrationsRequestQuery | FindDatasetsRequestQuery,
+  props: string[]
+) {
+  return mapValues(obj, (val, key) => (props.includes(key) ? JSON.stringify(val) : val));
+}
+
+/**
+ * Format the integrations and data streams search request into the required API format
+ */
+export const formatSearch = (search: FindIntegrationsRequestQuery | FindDatasetsRequestQuery) => {
+  return stringifyByProp(omitBy(search, isEmpty), ['searchAfter']);
+};

x-pack/plugins/discover_log_explorer/common/datasets/v1/find_datasets.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as rt from 'io-ts';
+import { datasetRT } from '../types';
+import { sortOrderRT } from './common';
+
+export const findDatasetsResponseRT = rt.type({
+  items: rt.array(datasetRT),
+});
+
+export const findDatasetsRequestQueryRT = rt.exact(
+  rt.partial({
+    datasetQuery: rt.string,
+    type: rt.literal('logs'),
+    sortOrder: sortOrderRT,
+    uncategorisedOnly: rt.boolean,
+  })
+);
+
+export type FindDatasetsRequestQuery = rt.TypeOf<typeof findDatasetsRequestQueryRT>;
+export type FindDatasetsResponse = rt.TypeOf<typeof findDatasetsResponseRT>;