[Discover] Add logs source and document contexts

.github/CODEOWNERS

@@ -373,6 +373,8 @@ examples/developer_examples @elastic/appex-sharedux
 examples/discover_customization_examples @elastic/kibana-data-discovery
 x-pack/plugins/discover_enhanced @elastic/kibana-data-discovery
 src/plugins/discover @elastic/kibana-data-discovery
+src/plugins/discover/public/context_awareness/profile_providers/log_document_profile @elastic/kibana-data-discovery @elastic/obs-ux-logs-team
+src/plugins/discover/public/context_awareness/profile_providers/logs_data_source_profile @elastic/kibana-data-discovery @elastic/obs-ux-logs-team
 src/plugins/discover_shared @elastic/kibana-data-discovery @elastic/obs-ux-logs-team
 packages/kbn-discover-utils @elastic/kibana-data-discovery
 packages/kbn-doc-links @elastic/docs

packages/kbn-data-view-utils/index.ts

@@ -7,3 +7,6 @@
  */
 
 export * from './src/constants';
+
+export { createRegExpPatternFrom } from './src/utils/create_regexp_pattern_from';
+export { testPatternAgainstAllowedList } from './src/utils/test_pattern_against_allowed_list';

packages/kbn-data-view-utils/src/utils/create_regexp_pattern_from.test.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { createRegExpPatternFrom } from './create_regexp_pattern_from';
+
+describe('createRegExpPatternFrom should create a regular expression starting from a string that', () => {
+  const regExpPattern = createRegExpPatternFrom('logs');
+
+  it('tests positive for single index patterns starting with the passed base pattern', () => {
+    expect('logs*').toMatch(regExpPattern);
+    expect('logs-*').toMatch(regExpPattern);
+    expect('logs-*-*').toMatch(regExpPattern);
+    expect('logs-system.syslog-*').toMatch(regExpPattern);
+
+    expect('logss*').not.toMatch(regExpPattern);
+    expect('logss-*').not.toMatch(regExpPattern);
+    expect('metrics*').not.toMatch(regExpPattern);
+    expect('metrics-*').not.toMatch(regExpPattern);
+  });
+
+  it('tests positive for single index patterns containing the passed base pattern', () => {
+    expect('foo-logs*').toMatch(regExpPattern);
+    expect('foo-logs-*').toMatch(regExpPattern);
+    expect('foo-logs-*-*').toMatch(regExpPattern);
+    expect('foo-logs-system.syslog-*').toMatch(regExpPattern);
+
+    expect('foo-logss*').not.toMatch(regExpPattern);
+    expect('foo-logss-*').not.toMatch(regExpPattern);
+    expect('foo-metrics*').not.toMatch(regExpPattern);
+    expect('foo-metrics-*').not.toMatch(regExpPattern);
+  });
+
+  it('tests positive for single index patterns with CCS prefixes', () => {
+    expect('cluster1:logs-*').toMatch(regExpPattern);
+    expect('cluster1:logs-*-*').toMatch(regExpPattern);
+    expect('cluster1:logs-system.syslog-*').toMatch(regExpPattern);
+    expect('cluster1:logs-system.syslog-default').toMatch(regExpPattern);
+
+    expect('cluster1:logss*').not.toMatch(regExpPattern);
+    expect('cluster1:logss-*').not.toMatch(regExpPattern);
+    expect('cluster1:metrics*').not.toMatch(regExpPattern);
+    expect('cluster1:metrics-*').not.toMatch(regExpPattern);
+  });
+
+  it('tests positive for multiple index patterns comma-separated if all of them individually match the criteria', () => {
+    expect('logs-*,cluster1:logs-*').toMatch(regExpPattern);
+    expect('cluster1:logs-*,cluster2:logs-*').toMatch(regExpPattern);
+    expect('*:logs-system.syslog-*,*:logs-system.errors-*').toMatch(regExpPattern);
+
+    expect('*:metrics-system.syslog-*,logs-system.errors-*').not.toMatch(regExpPattern);
+  });
+
+  it('tests positive for patterns with trialing commas', () => {
+    expect('logs-*,').toMatch(regExpPattern);
+    expect('cluster1:logs-*,logs-*,').toMatch(regExpPattern);
+  });
+
+  it('tests negative for patterns with spaces and unexpected commas', () => {
+    expect('cluster1:logs-*,clust,er2:logs-*').not.toMatch(regExpPattern);
+    expect('cluster1:logs-*,    cluster2:logs-*').not.toMatch(regExpPattern);
+  });
+});

x-pack/plugins/observability_solution/logs_explorer/common/data_views/utils.ts → packages/kbn-data-view-utils/src/utils/create_regexp_pattern_from.ts

@@ -1,15 +1,17 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-export const buildIndexPatternRegExp = (basePatterns: string[]) => {
+export const createRegExpPatternFrom = (basePatterns: string | string[]) => {
+  const patterns = Array.isArray(basePatterns) ? basePatterns : [basePatterns];
   // Create the base patterns union with strict boundaries
-  const basePatternGroup = `\\b(${basePatterns.join('|')})\\b([^,\\s]+)?`;
+  const basePatternGroup = `[^,\\s]*\\b(${patterns.join('|')})\\b([^,\\s]*)?`;
   // Apply base patterns union for local and remote clusters
-  const localAndRemotePatternGroup = `((${basePatternGroup})|([^:,\\s]+:${basePatternGroup}))`;
+  const localAndRemotePatternGroup = `((${basePatternGroup})|([^:,\\s]*:${basePatternGroup}))`;
   // Handle trailing comma and multiple pattern concatenation
   return new RegExp(`^${localAndRemotePatternGroup}(,${localAndRemotePatternGroup})*(,$|$)`, 'i');
 };

packages/kbn-data-view-utils/src/utils/test_pattern_against_allowed_list.test.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { testPatternAgainstAllowedList } from './test_pattern_against_allowed_list';
+
+describe('testPatternAgainstAllowedList', () => {
+  const allowedList = ['foo-logs-bar', /^\b(logs)\b([^,\s]*)/i];
+
+  it('should return true if the passed input matches any string or regexp of the passed list', () => {
+    expect(testPatternAgainstAllowedList(allowedList)('logs-*')).toBeTruthy();
+    expect(testPatternAgainstAllowedList(allowedList)('logs-*-*')).toBeTruthy();
+    expect(testPatternAgainstAllowedList(allowedList)('logs-system.syslog-*')).toBeTruthy();
+    expect(testPatternAgainstAllowedList(allowedList)('foo-logs-bar')).toBeTruthy();
+
+    expect(testPatternAgainstAllowedList(allowedList)('logss-*')).toBeFalsy();
+    expect(testPatternAgainstAllowedList(allowedList)('metrics*')).toBeFalsy();
+    expect(testPatternAgainstAllowedList(allowedList)('metrics-*')).toBeFalsy();
+  });
+});

packages/kbn-data-view-utils/src/utils/test_pattern_against_allowed_list.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const testPatternAgainstAllowedList =
+  (allowedList: Array<string | RegExp>) => (value: string) => {
+    for (const allowedItem of allowedList) {
+      const isMatchingString = typeof allowedItem === 'string' && value === allowedItem;
+      const isMatchingRegExp = allowedItem instanceof RegExp && allowedItem.test(value);
+
+      if (isMatchingString || isMatchingRegExp) {
+        return true;
+      }
+    }
+
+    // If no match is found in the allowedList, return false
+    return false;
+  };

packages/kbn-discover-utils/index.ts

@@ -30,6 +30,7 @@ export {
   IgnoredReason,
   buildDataTableRecord,
   buildDataTableRecordList,
+  createLogsContextService,
   fieldConstants,
   formatFieldValue,
   formatHit,
@@ -43,4 +44,6 @@ export {
   usePager,
 } from './src';
 
+export type { LogsContextService } from './src';
+
 export * from './src/types';

packages/kbn-discover-utils/src/data_types/logs/index.ts

@@ -8,3 +8,5 @@
 
 export * from './types';
 export * from './utils';
+
+export * from './logs_context_service';

packages/kbn-discover-utils/src/data_types/logs/logs_context_service.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { createRegExpPatternFrom, testPatternAgainstAllowedList } from '@kbn/data-view-utils';
+
+export interface LogsContextService {
+  isLogsIndexPattern(indexPattern: unknown): boolean;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface LogsContextServiceDeps {
+  // We will probably soon add uiSettings as a dependency
+  // to consume user configured indices
+}
+
+export const createLogsContextService = (_deps: LogsContextServiceDeps = {}) => {
+  // This is initially an hard-coded set of well-known base patterns,
+  // we can extend this allowed list with any setting coming from uiSettings
+  const ALLOWED_LOGS_DATA_SOURCES = [
+    createRegExpPatternFrom(['log', 'logs', 'logstash', 'auditbeat', 'filebeat', 'winlogbeat']),
+  ];
+
+  const isLogsIndexPattern = (indexPattern: unknown) => {
+    return (
+      typeof indexPattern === 'string' &&
+      testPatternAgainstAllowedList(ALLOWED_LOGS_DATA_SOURCES)(indexPattern)
+    );
+  };
+
+  return {
+    isLogsIndexPattern,
+  };
+};

packages/kbn-discover-utils/tsconfig.json

@@ -18,6 +18,7 @@
   "kbn_references": [
     "@kbn/data-service",
     "@kbn/data-views-plugin",
+    "@kbn/data-view-utils",
     "@kbn/es-query",
     "@kbn/field-formats-plugin",
     "@kbn/field-types",

src/plugins/discover/common/data_sources/utils.ts

@@ -25,3 +25,11 @@ export const isDataSourceType = <T extends DataSourceType>(
   dataSource: DiscoverDataSource | undefined,
   type: T
 ): dataSource is Extract<DiscoverDataSource, { type: T }> => dataSource?.type === type;
+
+export const isDataViewSource = (
+  dataSource: DiscoverDataSource | undefined
+): dataSource is DataViewDataSource => isDataSourceType(dataSource, DataSourceType.DataView);
+
+export const isEsqlSource = (
+  dataSource: DiscoverDataSource | undefined
+): dataSource is EsqlDataSource => isDataSourceType(dataSource, DataSourceType.Esql);

src/plugins/discover/public/build_services.ts

@@ -56,7 +56,7 @@ import { memoize, noop } from 'lodash';
 import type { NoDataPagePluginStart } from '@kbn/no-data-page-plugin/public';
 import type { AiopsPluginStart } from '@kbn/aiops-plugin/public';
 import type { DataVisualizerPluginStart } from '@kbn/data-visualizer-plugin/public';
-import type { DiscoverStartPlugins } from './plugin';
+import type { DiscoverStartPlugins } from './types';
 import type { DiscoverContextAppLocator } from './application/context/services/locator';
 import type { DiscoverSingleDocLocator } from './application/doc/locator';
 import type { DiscoverAppLocator } from '../common';

src/plugins/discover/public/context_awareness/__mocks__/index.ts

@@ -19,6 +19,7 @@ import {
   RootProfileService,
   SolutionType,
 } from '../profiles';
+import { createProfileProviderServices } from '../profiles/profile_provider_services';
 import { ProfilesManager } from '../profiles_manager';
 
 export const createContextAwarenessMocks = () => {
@@ -89,12 +90,15 @@ export const createContextAwarenessMocks = () => {
     documentProfileServiceMock
   );
 
+  const profileProviderServices = createProfileProviderServices();
+
   return {
     rootProfileProviderMock,
     dataSourceProfileProviderMock,
     documentProfileProviderMock,
     contextRecordMock,
     contextRecordMock2,
     profilesManagerMock,
+    profileProviderServices,
   };
 };

src/plugins/discover/public/context_awareness/profile_providers/log_document_profile/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { createLogDocumentProfileProvider } from './profile';

src/plugins/discover/public/context_awareness/profile_providers/log_document_profile/profile.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { buildDataTableRecord } from '@kbn/discover-utils';
+import { DocumentType } from '../../profiles';
+import { createContextAwarenessMocks } from '../../__mocks__';
+import { createLogDocumentProfileProvider } from './profile';
+
+const mockServices = createContextAwarenessMocks().profileProviderServices;
+
+describe('logDocumentProfileProvider', () => {
+  const logDocumentProfileProvider = createLogDocumentProfileProvider(mockServices);
+  const RESOLUTION_MATCH = {
+    isMatch: true,
+    context: {
+      type: DocumentType.Log,
+    },
+  };
+  const RESOLUTION_MISMATCH = {
+    isMatch: false,
+  };
+
+  it('matches records with the correct data stream type', () => {
+    expect(
+      logDocumentProfileProvider.resolve({
+        record: buildMockRecord('logs-2000-01-01', {
+          'data_stream.type': ['logs'],
+        }),
+      })
+    ).toEqual(RESOLUTION_MATCH);
+  });
+
+  it('matches records with fields prefixed with "log."', () => {
+    expect(
+      logDocumentProfileProvider.resolve({
+        record: buildMockRecord('logs-2000-01-01', {
+          'log.level': ['INFO'],
+        }),
+      })
+    ).toEqual(RESOLUTION_MATCH);
+  });
+
+  it('matches records with indices matching the allowed pattern', () => {
+    expect(
+      logDocumentProfileProvider.resolve({
+        record: buildMockRecord('logs-2000-01-01'),
+      })
+    ).toEqual(RESOLUTION_MATCH);
+    expect(
+      logDocumentProfileProvider.resolve({
+        record: buildMockRecord('remote_cluster:filebeat'),
+      })
+    ).toEqual(RESOLUTION_MATCH);
+  });
+
+  it('does not match records with neither characteristic', () => {
+    expect(
+      logDocumentProfileProvider.resolve({
+        record: buildMockRecord('another-index'),
+      })
+    ).toEqual(RESOLUTION_MISMATCH);
+  });
+});
+
+const buildMockRecord = (index: string, fields: Record<string, unknown[]> = {}) =>
+  buildDataTableRecord({
+    _id: '',
+    _index: index,
+    fields: {
+      _index: index,
+      ...fields,
+    },
+  });