[Endpoint][EPM] Retrieve Index Pattern from Ingest Manager

.github/CODEOWNERS

@@ -202,9 +202,12 @@
 # Endpoint
 /x-pack/plugins/endpoint/ @elastic/endpoint-app-team
 /x-pack/test/api_integration/apis/endpoint/ @elastic/endpoint-app-team
+/x-pack/test/endpoint_api_integration_no_ingest/ @elastic/endpoint-app-team
 /x-pack/test/functional_endpoint/ @elastic/endpoint-app-team
 /x-pack/test/functional_endpoint_ingest_failure/ @elastic/endpoint-app-team
 /x-pack/test/functional/es_archives/endpoint/ @elastic/endpoint-app-team
+/x-pack/test/plugin_functional/plugins/resolver_test/ @elastic/endpoint-app-team
+/x-pack/test/plugin_functional/test_suites/resolver/ @elastic/endpoint-app-team
 
 # SIEM
 /x-pack/legacy/plugins/siem/ @elastic/siem

x-pack/plugins/endpoint/common/schema/index_pattern.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+export const indexPatternGetParamsSchema = schema.object({ datasetPath: schema.string() });

x-pack/plugins/endpoint/common/types.ts

@@ -7,6 +7,7 @@
 import { SearchResponse } from 'elasticsearch';
 import { TypeOf } from '@kbn/config-schema';
 import { alertingIndexGetQuerySchema } from './schema/alert_index';
+import { indexPatternGetParamsSchema } from './schema/index_pattern';
 
 /**
  * A deep readonly type that will make all children of a given object readonly recursively
@@ -30,9 +31,9 @@ export type Direction = 'asc' | 'desc';
 
 export class EndpointAppConstants {
   static BASE_API_URL = '/api/endpoint';
-  static ENDPOINT_INDEX_NAME = 'endpoint-agent*';
+  static INDEX_PATTERN_ROUTE = `${EndpointAppConstants.BASE_API_URL}/index_pattern`;
   static ALERT_INDEX_NAME = 'events-endpoint-1';
-  static EVENT_INDEX_NAME = 'events-endpoint-*';
+  static EVENT_DATASET = 'events';
   static DEFAULT_TOTAL_HITS = 10000;
   /**
    * Legacy events are stored in indices with endgame-* prefix
@@ -442,3 +443,8 @@ export type AlertingIndexGetQueryInput = KbnConfigSchemaInputTypeOf<
  * Result of the validated query params when handling alert index requests.
  */
 export type AlertingIndexGetQueryResult = TypeOf<typeof alertingIndexGetQuerySchema>;
+
+/**
+ * Result of the validated params when handling an index pattern request.
+ */
+export type IndexPatternGetParamsResult = TypeOf<typeof indexPatternGetParamsSchema>;

x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts

@@ -15,10 +15,14 @@ import { EndpointAppConstants } from '../../../../../common/types';
 export const alertMiddlewareFactory: MiddlewareFactory<AlertListState> = (coreStart, depsStart) => {
   async function fetchIndexPatterns(): Promise<IIndexPattern[]> {
     const { indexPatterns } = depsStart.data;
-    const indexName = EndpointAppConstants.ALERT_INDEX_NAME;
-    const fields = await indexPatterns.getFieldsForWildcard({ pattern: indexName });
+    const eventsPattern: { indexPattern: string } = await coreStart.http.get(
+      `${EndpointAppConstants.INDEX_PATTERN_ROUTE}/${EndpointAppConstants.EVENT_DATASET}`
+    );
+    const fields = await indexPatterns.getFieldsForWildcard({
+      pattern: eventsPattern.indexPattern,
+    });
     const indexPattern: IIndexPattern = {
-      title: indexName,
+      title: eventsPattern.indexPattern,
       fields,
     };
 

x-pack/plugins/endpoint/scripts/resolver_generator.ts

@@ -35,13 +35,13 @@ async function main() {
     eventIndex: {
       alias: 'ei',
       describe: 'index to store events in',
-      default: 'events-endpoint-1',
+      default: 'events-endpoint-default-1',
       type: 'string',
     },
     metadataIndex: {
       alias: 'mi',
       describe: 'index to store host metadata in',
-      default: 'endpoint-agent-1',
+      default: 'metrics-endpoint-default-1',
       type: 'string',
     },
     auth: {

x-pack/plugins/endpoint/server/index_pattern.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { Logger, LoggerFactory, RequestHandlerContext } from 'kibana/server';
+import { ESIndexPatternService } from '../../ingest_manager/server';
+import { EndpointAppConstants } from '../common/types';
+
+export interface IndexPatternRetriever {
+  getIndexPattern(ctx: RequestHandlerContext, datasetPath: string): Promise<string>;
+  getEventIndexPattern(ctx: RequestHandlerContext): Promise<string>;
+  getMetadataIndexPattern(ctx: RequestHandlerContext): Promise<string>;
+}
+
+export class IngestIndexPatternRetriever implements IndexPatternRetriever {
+  private static endpointPackageName = 'endpoint';
+  private static metadataDataset = 'metadata';
+  private readonly log: Logger;
+  constructor(private readonly service: ESIndexPatternService, loggerFactory: LoggerFactory) {
+    this.log = loggerFactory.get('index-pattern-retriever');
+  }
+
+  async getEventIndexPattern(ctx: RequestHandlerContext) {
+    return await this.getIndexPattern(ctx, EndpointAppConstants.EVENT_DATASET);
+  }
+
+  async getMetadataIndexPattern(ctx: RequestHandlerContext) {
+    return await this.getIndexPattern(ctx, IngestIndexPatternRetriever.metadataDataset);
+  }
+
+  async getIndexPattern(ctx: RequestHandlerContext, datasetPath: string) {
+    const pattern = await this.service.getESIndexPattern(
+      ctx.core.savedObjects.client,
+      IngestIndexPatternRetriever.endpointPackageName,
+      datasetPath
+    );
+
+    if (!pattern) {
+      const msg = `Unable to retrieve the index pattern for dataset: ${datasetPath}`;
+      this.log.warn(msg);
+      throw new Error(msg);
+    }
+
+    return pattern;
+  }
+}

x-pack/plugins/endpoint/server/mocks.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const createMockIndexPatternRetriever = (indexPattern: string) => {
+  const mockGetFunc = jest.fn().mockResolvedValue(indexPattern);
+  return {
+    getIndexPattern: mockGetFunc,
+    getEventIndexPattern: mockGetFunc,
+    getMetadataIndexPattern: mockGetFunc,
+  };
+};
+
+export const MetadataIndexPattern = 'metrics-endpoint-*';
+
+export const createMockMetadataIPRetriever = () => {
+  return createMockIndexPatternRetriever(MetadataIndexPattern);
+};
+
+export const createMockIndexPatternService = (indexPattern: string) => {
+  return {
+    esIndexPatternService: {
+      getESIndexPattern: jest.fn().mockResolvedValue(indexPattern),
+    },
+  };
+};

x-pack/plugins/endpoint/server/plugin.test.ts

@@ -7,6 +7,7 @@
 import { EndpointPlugin, EndpointPluginSetupDependencies } from './plugin';
 import { coreMock } from '../../../../src/core/server/mocks';
 import { PluginSetupContract } from '../../features/server';
+import { createMockIndexPatternService } from './mocks';
 
 describe('test endpoint plugin', () => {
   let plugin: EndpointPlugin;
@@ -28,7 +29,10 @@ describe('test endpoint plugin', () => {
       getFeaturesUICapabilities: jest.fn(),
       registerLegacyAPI: jest.fn(),
     };
-    mockedEndpointPluginSetupDependencies = { features: mockedPluginSetupContract };
+    mockedEndpointPluginSetupDependencies = {
+      features: mockedPluginSetupContract,
+      ingestManager: createMockIndexPatternService(''),
+    };
   });
 
   it('test properly setup plugin', async () => {

x-pack/plugins/endpoint/server/plugin.ts

@@ -6,19 +6,23 @@
 import { Plugin, CoreSetup, PluginInitializerContext, Logger } from 'kibana/server';
 import { first } from 'rxjs/operators';
 import { PluginSetupContract as FeaturesPluginSetupContract } from '../../features/server';
+import { IngestManagerSetupContract } from '../../ingest_manager/server';
 import { createConfig$, EndpointConfigType } from './config';
 import { EndpointAppContext } from './types';
 
 import { registerAlertRoutes } from './routes/alerts';
 import { registerResolverRoutes } from './routes/resolver';
+import { registerIndexPatternRoute } from './routes/index_pattern';
 import { registerEndpointRoutes } from './routes/metadata';
+import { IngestIndexPatternRetriever } from './index_pattern';
 
 export type EndpointPluginStart = void;
 export type EndpointPluginSetup = void;
 export interface EndpointPluginStartDependencies {} // eslint-disable-line @typescript-eslint/no-empty-interface
 
 export interface EndpointPluginSetupDependencies {
   features: FeaturesPluginSetupContract;
+  ingestManager: IngestManagerSetupContract;
 }
 
 export class EndpointPlugin
@@ -62,6 +66,10 @@ export class EndpointPlugin
       },
     });
     const endpointContext = {
+      indexPatternRetriever: new IngestIndexPatternRetriever(
+        plugins.ingestManager.esIndexPatternService,
+        this.initializerContext.logger
+      ),
       logFactory: this.initializerContext.logger,
       config: (): Promise<EndpointConfigType> => {
         return createConfig$(this.initializerContext)
@@ -73,6 +81,7 @@ export class EndpointPlugin
     registerEndpointRoutes(router, endpointContext);
     registerResolverRoutes(router, endpointContext);
     registerAlertRoutes(router, endpointContext);
+    registerIndexPatternRoute(router, endpointContext);
   }
 
   public start() {

x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts

@@ -12,6 +12,7 @@ import {
 import { registerAlertRoutes } from './index';
 import { EndpointConfigSchema } from '../../config';
 import { alertingIndexGetQuerySchema } from '../../../common/schema/alert_index';
+import { createMockIndexPatternRetriever } from '../../mocks';
 
 describe('test alerts route', () => {
   let routerMock: jest.Mocked<IRouter>;
@@ -24,6 +25,7 @@ describe('test alerts route', () => {
     mockClusterClient.asScoped.mockReturnValue(mockScopedClient);
     routerMock = httpServiceMock.createRouter();
     registerAlertRoutes(routerMock, {
+      indexPatternRetriever: createMockIndexPatternRetriever('events-endpoint-*'),
       logFactory: loggingServiceMock.create(),
       config: () => Promise.resolve(EndpointConfigSchema.validate({})),
     });

x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts

@@ -22,22 +22,29 @@ export const alertDetailsHandlerWrapper = function(
     try {
       const alertId = req.params.id;
       const response = (await ctx.core.elasticsearch.dataClient.callAsCurrentUser('get', {
+        // Remove the hard coded reference to an alert index here
+        // https://github.com/elastic/endpoint-app-team/issues/311
         index: EndpointAppConstants.ALERT_INDEX_NAME,
         id: alertId,
       })) as GetResponse<AlertEvent>;
 
+      const indexPattern = await endpointAppContext.indexPatternRetriever.getEventIndexPattern(ctx);
+
       const config = await endpointAppContext.config();
       const pagination: AlertDetailsPagination = new AlertDetailsPagination(
         config,
         ctx,
         req.params,
-        response
+        response,
+        indexPattern
       );
 
-      const currentHostInfo = await getHostData(ctx, response._source.host.id);
+      const currentHostInfo = await getHostData(ctx, response._source.host.id, indexPattern);
 
       return res.ok({
         body: {
+          // base64 encode the index in the response
+          // https://github.com/elastic/endpoint-app-team/issues/311
           id: response._id,
           ...response._source,
           state: {

x-pack/plugins/endpoint/server/routes/alerts/details/lib/pagination.ts

@@ -29,7 +29,8 @@ export class AlertDetailsPagination extends Pagination<
     config: EndpointConfigType,
     requestContext: RequestHandlerContext,
     state: AlertDetailsRequestParams,
-    data: GetResponse<AlertEvent>
+    data: GetResponse<AlertEvent>,
+    private readonly indexPattern: string
   ) {
     super(config, requestContext, state, data);
   }
@@ -54,7 +55,8 @@ export class AlertDetailsPagination extends Pagination<
 
     const response = await searchESForAlerts(
       this.requestContext.core.elasticsearch.dataClient,
-      reqData
+      reqData,
+      this.indexPattern
     );
     return response;
   }

x-pack/plugins/endpoint/server/routes/alerts/lib/index.ts

@@ -97,7 +97,8 @@ function buildSort(query: AlertSearchQuery): AlertSort {
  * Builds a request body for Elasticsearch, given a set of query params.
  **/
 const buildAlertSearchQuery = async (
-  query: AlertSearchQuery
+  query: AlertSearchQuery,
+  indexPattern: string
 ): Promise<AlertSearchRequestWrapper> => {
   let totalHitsMin: number = EndpointAppConstants.DEFAULT_TOTAL_HITS;
 
@@ -125,7 +126,7 @@ const buildAlertSearchQuery = async (
 
   const reqWrapper: AlertSearchRequestWrapper = {
     size: query.pageSize,
-    index: EndpointAppConstants.ALERT_INDEX_NAME,
+    index: indexPattern,
     body: reqBody,
   };
 
@@ -141,9 +142,10 @@ const buildAlertSearchQuery = async (
  **/
 export const searchESForAlerts = async (
   dataClient: IScopedClusterClient,
-  query: AlertSearchQuery
+  query: AlertSearchQuery,
+  indexPattern: string
 ): Promise<SearchResponse<AlertEvent>> => {
-  const reqWrapper = await buildAlertSearchQuery(query);
+  const reqWrapper = await buildAlertSearchQuery(query, indexPattern);
   const response = (await dataClient.callAsCurrentUser('search', reqWrapper)) as SearchResponse<
     AlertEvent
   >;

x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts

@@ -18,8 +18,13 @@ export const alertListHandlerWrapper = function(
     res
   ) => {
     try {
+      const indexPattern = await endpointAppContext.indexPatternRetriever.getEventIndexPattern(ctx);
       const reqData = await getRequestData(req, endpointAppContext);
-      const response = await searchESForAlerts(ctx.core.elasticsearch.dataClient, reqData);
+      const response = await searchESForAlerts(
+        ctx.core.elasticsearch.dataClient,
+        reqData,
+        indexPattern
+      );
       const mappedBody = await mapToAlertResultList(ctx, endpointAppContext, reqData, response);
       return res.ok({ body: mappedBody });
     } catch (err) {

x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts

@@ -108,6 +108,8 @@ export async function mapToAlertResultList(
 
   function mapHit(entry: AlertHits[0]): AlertData {
     return {
+      // base64 encode the alert index in the id field here
+      // https://github.com/elastic/endpoint-app-team/issues/311
       id: entry._id,
       ...entry._source,
       prev: null,

x-pack/plugins/endpoint/server/routes/index_pattern.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter, Logger, RequestHandler } from 'kibana/server';
+import { EndpointAppContext } from '../types';
+import { IndexPatternGetParamsResult, EndpointAppConstants } from '../../common/types';
+import { indexPatternGetParamsSchema } from '../../common/schema/index_pattern';
+import { IndexPatternRetriever } from '../index_pattern';
+
+function handleIndexPattern(
+  log: Logger,
+  indexRetriever: IndexPatternRetriever
+): RequestHandler<IndexPatternGetParamsResult> {
+  return async (context, req, res) => {
+    try {
+      return res.ok({
+        body: {
+          indexPattern: await indexRetriever.getIndexPattern(context, req.params.datasetPath),
+        },
+      });
+    } catch (error) {
+      log.warn(error);
+      return res.notFound({ body: error });
+    }
+  };
+}
+
+export function registerIndexPatternRoute(router: IRouter, endpointAppContext: EndpointAppContext) {
+  const log = endpointAppContext.logFactory.get('index_pattern');
+
+  router.get(
+    {
+      path: `${EndpointAppConstants.INDEX_PATTERN_ROUTE}/{datasetPath}`,
+      validate: { params: indexPatternGetParamsSchema },
+      options: { authRequired: true },
+    },
+    handleIndexPattern(log, endpointAppContext.indexPatternRetriever)
+  );
+}