From b69c2ada5f90a18914be0e23421fad398a666b95 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 7 Apr 2020 18:08:09 -0400 Subject: [PATCH 01/27] Endpoint successfully depending on ingest manager to initialize --- .github/CODEOWNERS | 1 + x-pack/plugins/endpoint/kibana.json | 2 +- .../public/applications/endpoint/index.tsx | 4 +- .../public/applications/endpoint/mocks.ts | 3 ++ .../applications/endpoint/view/setup.tsx | 52 +++++++++++++++++++ x-pack/plugins/endpoint/public/plugin.ts | 2 + .../ingest_manager/common/types/models/epm.ts | 1 + x-pack/plugins/ingest_manager/public/index.ts | 2 + .../plugins/ingest_manager/public/plugin.ts | 20 ++++++- x-pack/scripts/functional_tests.js | 1 + x-pack/test/api_integration/config.js | 3 -- .../functional/apps/endpoint/landing_page.ts | 7 ++- x-pack/test/functional/config.js | 2 + .../endpoint/alerts/api_feature/mappings.json | 3 +- .../apps/endpoint/index.ts | 14 +++++ .../apps/endpoint/landing_page.ts | 22 ++++++++ .../config.ts | 25 +++++++++ 17 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugins/endpoint/public/applications/endpoint/view/setup.tsx create mode 100644 x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/index.ts create mode 100644 x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/landing_page.ts create mode 100644 x-pack/test/functional_endpoint_ingest_failure/config.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index feaf47e45fd69..635f44c3596e2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -203,6 +203,7 @@ /x-pack/plugins/endpoint/ @elastic/endpoint-app-team /x-pack/test/api_integration/apis/endpoint/ @elastic/endpoint-app-team /x-pack/test/functional/apps/endpoint/ @elastic/endpoint-app-team +/x-pack/test/functional_endpoint_ingest_failure/ @elastic/endpoint-app-team /x-pack/test/functional/es_archives/endpoint/ @elastic/endpoint-app-team # SIEM diff --git a/x-pack/plugins/endpoint/kibana.json b/x-pack/plugins/endpoint/kibana.json index 5b8bec7777406..4b48c83fb0e7c 100644 --- a/x-pack/plugins/endpoint/kibana.json +++ b/x-pack/plugins/endpoint/kibana.json @@ -3,7 +3,7 @@ "version": "1.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "endpoint"], - "requiredPlugins": ["features", "embeddable", "data", "dataEnhanced"], + "requiredPlugins": ["features", "embeddable", "data", "dataEnhanced", "ingestManager"], "server": true, "ui": true } diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/index.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/index.tsx index fa9055e0d9bbd..fe35b1293c038 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/index.tsx +++ b/x-pack/plugins/endpoint/public/applications/endpoint/index.tsx @@ -22,6 +22,7 @@ import { PolicyList } from './view/policy'; import { PolicyDetails } from './view/policy'; import { HeaderNavigation } from './components/header_nav'; import { EuiThemeProvider } from '../../../../../legacy/common/eui_styled_components'; +import { Setup } from './view/setup'; /** * This module will be loaded asynchronously to reduce the bundle size of your plugin's main bundle. @@ -53,7 +54,7 @@ const AppRoot: React.FunctionComponent = React.memo( history, store, coreStart: { http, notifications, uiSettings, application }, - depsStart: { data }, + depsStart: { data, ingestManager }, }) => { const isDarkMode = useObservable(uiSettings.get$('theme:darkMode')); @@ -61,6 +62,7 @@ const AppRoot: React.FunctionComponent = React.memo( + diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/mocks.ts b/x-pack/plugins/endpoint/public/applications/endpoint/mocks.ts index e1a90b4a416dc..877366239ab43 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/mocks.ts +++ b/x-pack/plugins/endpoint/public/applications/endpoint/mocks.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { IngestManagerStart } from '../../../../ingest_manager/public'; import { dataPluginMock, Start as DataPublicStartMock, @@ -32,6 +33,7 @@ type DataMock = Omit & { */ export interface DepsStartMock { data: DataMock; + ingestManager: IngestManagerStart; } /** @@ -54,5 +56,6 @@ export const depsStartMock: () => DepsStartMock = () => { return { data: dataMock, + ingestManager: { success: true }, }; }; diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/view/setup.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/view/setup.tsx new file mode 100644 index 0000000000000..a826e1f30f75d --- /dev/null +++ b/x-pack/plugins/endpoint/public/applications/endpoint/view/setup.tsx @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import * as React from 'react'; +import { i18n } from '@kbn/i18n'; +import { NotificationsStart } from 'kibana/public'; +import { IngestManagerStart } from '../../../../../ingest_manager/public'; + +export const Setup: React.FunctionComponent<{ + ingestManager: IngestManagerStart; + notifications: NotificationsStart; +}> = ({ ingestManager, notifications }) => { + React.useEffect(() => { + const defaultText = i18n.translate('xpack.endpoint.ingestToastMessage', { + defaultMessage: 'Ingest Manager failed during its setup.', + }); + + const title = i18n.translate('xpack.endpoint.ingestToastTitle', { + defaultMessage: 'App failed to initialize', + }); + + const displayToastWithModal = (text: string) => { + const errorText = new Error(defaultText); + // we're leveraging the notification's error toast which is usually used for displaying stack traces of an + // actually Error. Instead of displaying a stack trace we'll display the more detailed error text when the + // user clicks `See the full error` button to see the modal + errorText.stack = text; + notifications.toasts.addError(errorText, { + title, + }); + }; + + const displayToast = () => { + notifications.toasts.addDanger({ + title, + text: defaultText, + }); + }; + + if (!ingestManager.success) { + if (ingestManager.error) { + displayToastWithModal(ingestManager.error.message); + } else { + displayToast(); + } + } + }, [ingestManager, notifications.toasts]); + + return null; +}; diff --git a/x-pack/plugins/endpoint/public/plugin.ts b/x-pack/plugins/endpoint/public/plugin.ts index ee5bbe71ae8aa..9964454add801 100644 --- a/x-pack/plugins/endpoint/public/plugin.ts +++ b/x-pack/plugins/endpoint/public/plugin.ts @@ -8,6 +8,7 @@ import { Plugin, CoreSetup, AppMountParameters, CoreStart } from 'kibana/public' import { EmbeddableSetup } from 'src/plugins/embeddable/public'; import { DataPublicPluginStart } from 'src/plugins/data/public'; import { i18n } from '@kbn/i18n'; +import { IngestManagerStart } from '../../ingest_manager/public'; import { ResolverEmbeddableFactory } from './embeddables/resolver'; export type EndpointPluginStart = void; @@ -18,6 +19,7 @@ export interface EndpointPluginSetupDependencies { } export interface EndpointPluginStartDependencies { data: DataPublicPluginStart; + ingestManager: IngestManagerStart; } /** diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index efa6621001038..5524e7505d74b 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -252,6 +252,7 @@ export enum IngestAssetType { export enum DefaultPackages { base = 'base', system = 'system', + endpoint = 'endpoint', } export interface IndexTemplate { diff --git a/x-pack/plugins/ingest_manager/public/index.ts b/x-pack/plugins/ingest_manager/public/index.ts index aa1e0e79e548b..c11ad60dffee4 100644 --- a/x-pack/plugins/ingest_manager/public/index.ts +++ b/x-pack/plugins/ingest_manager/public/index.ts @@ -6,6 +6,8 @@ import { PluginInitializerContext } from 'src/core/public'; import { IngestManagerPlugin } from './plugin'; +export { IngestManagerStart } from './plugin'; + export const plugin = (initializerContext: PluginInitializerContext) => { return new IngestManagerPlugin(initializerContext); }; diff --git a/x-pack/plugins/ingest_manager/public/plugin.ts b/x-pack/plugins/ingest_manager/public/plugin.ts index d7be1c1f1fe6e..77bba0bb0f990 100644 --- a/x-pack/plugins/ingest_manager/public/plugin.ts +++ b/x-pack/plugins/ingest_manager/public/plugin.ts @@ -17,11 +17,20 @@ import { LicensingPluginSetup } from '../../licensing/public'; import { PLUGIN_ID } from '../common/constants'; import { IngestManagerConfigType } from '../common/types'; +import { setupRouteService } from '../common'; export { IngestManagerConfigType } from '../common/types'; export type IngestManagerSetup = void; -export type IngestManagerStart = void; +/** + * Describes public IngestManager plugin contract returned at the `start` stage. + */ +export interface IngestManagerStart { + success: boolean; + error?: { + message: string; + }; +} export interface IngestManagerSetupDeps { licensing: LicensingPluginSetup; @@ -61,7 +70,14 @@ export class IngestManagerPlugin }); } - public start(core: CoreStart) {} + public async start(core: CoreStart): Promise { + try { + const { isInitialized: success } = await core.http.post(setupRouteService.getSetupPath()); + return { success }; + } catch (error) { + return { success: false, error: { message: error.body?.message || 'Unknown error' } }; + } + } public stop() {} } diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index 7943da07716a1..294042e39cd6c 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -43,6 +43,7 @@ const onlyNotInCoverageTests = [ require.resolve('../test/licensing_plugin/config.ts'), require.resolve('../test/licensing_plugin/config.public.ts'), require.resolve('../test/licensing_plugin/config.legacy.ts'), + require.resolve('../test/functional_endpoint_ingest_failure/config.ts'), ]; require('@kbn/plugin-helpers').babelRegister(); diff --git a/x-pack/test/api_integration/config.js b/x-pack/test/api_integration/config.js index b62368bf2d608..ab30b1fe16ddd 100644 --- a/x-pack/test/api_integration/config.js +++ b/x-pack/test/api_integration/config.js @@ -26,9 +26,6 @@ export async function getApiIntegrationConfig({ readConfigFile }) { ...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), '--xpack.security.session.idleTimeout=3600000', // 1 hour '--optimize.enabled=false', - '--xpack.endpoint.enabled=true', - '--xpack.ingestManager.enabled=true', - '--xpack.ingestManager.fleet.enabled=true', '--xpack.endpoint.alertResultListDefaultDateRange.from=2018-01-10T00:00:00.000Z', ], }, diff --git a/x-pack/test/functional/apps/endpoint/landing_page.ts b/x-pack/test/functional/apps/endpoint/landing_page.ts index 65af91feae407..b4da4631aa60b 100644 --- a/x-pack/test/functional/apps/endpoint/landing_page.ts +++ b/x-pack/test/functional/apps/endpoint/landing_page.ts @@ -7,8 +7,9 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; -export default ({ getPageObjects }: FtrProviderContext) => { +export default ({ getPageObjects, getService }: FtrProviderContext) => { const pageObjects = getPageObjects(['common', 'endpoint']); + const testSubjects = getService('testSubjects'); describe('Endpoint landing page', function() { this.tags('ciGroup7'); @@ -20,5 +21,9 @@ export default ({ getPageObjects }: FtrProviderContext) => { const welcomeEndpointMessage = await pageObjects.endpoint.welcomeEndpointTitle(); expect(welcomeEndpointMessage).to.be('Hello World'); }); + + it('Does not display a toast indicating that the ingest manager failed to initialize', async () => { + await testSubjects.missingOrFail('euiToastHeader'); + }); }); }; diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index cff555feace18..4143effcbf233 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -89,6 +89,8 @@ export default async function({ readConfigFile }) { '--telemetry.banner=false', '--timelion.ui.enabled=true', '--xpack.endpoint.enabled=true', + '--xpack.ingestManager.enabled=true', + '--xpack.ingestManager.fleet.enabled=true', ], }, uiSettings: { diff --git a/x-pack/test/functional/es_archives/endpoint/alerts/api_feature/mappings.json b/x-pack/test/functional/es_archives/endpoint/alerts/api_feature/mappings.json index 64dc395ab69a4..7068c24a4b26c 100644 --- a/x-pack/test/functional/es_archives/endpoint/alerts/api_feature/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/alerts/api_feature/mappings.json @@ -389,7 +389,8 @@ "type": "nested" }, "file_extension": { - "type": "long" + "ignore_above": 1024, + "type": "keyword" }, "project_file": { "properties": { diff --git a/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/index.ts b/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/index.ts new file mode 100644 index 0000000000000..068880d3a8433 --- /dev/null +++ b/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; + +export default function({ loadTestFile }: FtrProviderContext) { + describe('endpoint when the ingest manager fails to setup correctly', function() { + this.tags('ciGroup7'); + + loadTestFile(require.resolve('./landing_page')); + }); +} diff --git a/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/landing_page.ts b/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/landing_page.ts new file mode 100644 index 0000000000000..c5f3016c15394 --- /dev/null +++ b/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/landing_page.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; + +export default ({ getPageObjects, getService }: FtrProviderContext) => { + describe('home page', function() { + const pageObjects = getPageObjects(['common', 'endpoint']); + const testSubjects = getService('testSubjects'); + + before(async () => { + await pageObjects.common.navigateToApp('endpoint'); + }); + + it('displays an error toast', async () => { + await testSubjects.existOrFail('euiToastHeader'); + }); + }); +}; diff --git a/x-pack/test/functional_endpoint_ingest_failure/config.ts b/x-pack/test/functional_endpoint_ingest_failure/config.ts new file mode 100644 index 0000000000000..2d825a5a423a8 --- /dev/null +++ b/x-pack/test/functional_endpoint_ingest_failure/config.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { resolve } from 'path'; +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; + +export default async function({ readConfigFile }: FtrConfigProviderContext) { + const xpackFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js')); + + return { + ...xpackFunctionalConfig.getAll(), + testFiles: [resolve(__dirname, './apps/endpoint')], + kbnTestServer: { + ...xpackFunctionalConfig.get('kbnTestServer'), + serverArgs: [ + ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'), + // use a bogus port so the ingest manager setup will fail + '--xpack.ingestManager.epm.registryUrl=http://127.0.0.1:12345', + ], + }, + }; +} From c9f050744aa367c79d7c2cc3a0d04b4e06290171 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 10:07:36 -0400 Subject: [PATCH 02/27] Moving the endpoint functional tests to their own directory to avoid enabling ingest in the base tests --- .../endpoint/mocks/dependencies_start_mock.ts | 2 +- x-pack/scripts/functional_tests.js | 1 + x-pack/test/api_integration/config.js | 3 +++ x-pack/test/functional/config.js | 3 --- .../apps/endpoint/alerts.ts | 2 +- .../feature_controls/endpoint_spaces.ts | 2 +- .../apps/endpoint/feature_controls/index.ts | 2 +- .../apps/endpoint/header_nav.ts | 2 +- .../apps/endpoint/host_list.ts | 2 +- .../apps/endpoint/index.ts | 2 +- .../apps/endpoint/landing_page.ts | 2 +- .../apps/endpoint/policy_list.ts | 2 +- x-pack/test/functional_endpoint/config.ts | 26 +++++++++++++++++++ .../config.ts | 3 +++ 14 files changed, 42 insertions(+), 12 deletions(-) rename x-pack/test/{functional => functional_endpoint}/apps/endpoint/alerts.ts (97%) rename x-pack/test/{functional => functional_endpoint}/apps/endpoint/feature_controls/endpoint_spaces.ts (96%) rename x-pack/test/{functional => functional_endpoint}/apps/endpoint/feature_controls/index.ts (84%) rename x-pack/test/{functional => functional_endpoint}/apps/endpoint/header_nav.ts (96%) rename x-pack/test/{functional => functional_endpoint}/apps/endpoint/host_list.ts (98%) rename x-pack/test/{functional => functional_endpoint}/apps/endpoint/index.ts (89%) rename x-pack/test/{functional => functional_endpoint}/apps/endpoint/landing_page.ts (92%) rename x-pack/test/{functional => functional_endpoint}/apps/endpoint/policy_list.ts (95%) create mode 100644 x-pack/test/functional_endpoint/config.ts diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/mocks/dependencies_start_mock.ts b/x-pack/plugins/endpoint/public/applications/endpoint/mocks/dependencies_start_mock.ts index dce63807c65f2..d3fc653f4c9ba 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/mocks/dependencies_start_mock.ts +++ b/x-pack/plugins/endpoint/public/applications/endpoint/mocks/dependencies_start_mock.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IngestManagerStart } from '../../../../ingest_manager/public'; +import { IngestManagerStart } from '../../../../../ingest_manager/public'; import { dataPluginMock, Start as DataPublicStartMock, diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index 294042e39cd6c..061c9e4a0d921 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -44,6 +44,7 @@ const onlyNotInCoverageTests = [ require.resolve('../test/licensing_plugin/config.public.ts'), require.resolve('../test/licensing_plugin/config.legacy.ts'), require.resolve('../test/functional_endpoint_ingest_failure/config.ts'), + require.resolve('../test/functional_endpoint/config.ts'), ]; require('@kbn/plugin-helpers').babelRegister(); diff --git a/x-pack/test/api_integration/config.js b/x-pack/test/api_integration/config.js index ab30b1fe16ddd..b62368bf2d608 100644 --- a/x-pack/test/api_integration/config.js +++ b/x-pack/test/api_integration/config.js @@ -26,6 +26,9 @@ export async function getApiIntegrationConfig({ readConfigFile }) { ...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), '--xpack.security.session.idleTimeout=3600000', // 1 hour '--optimize.enabled=false', + '--xpack.endpoint.enabled=true', + '--xpack.ingestManager.enabled=true', + '--xpack.ingestManager.fleet.enabled=true', '--xpack.endpoint.alertResultListDefaultDateRange.from=2018-01-10T00:00:00.000Z', ], }, diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index 4143effcbf233..b45577da9f282 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -88,9 +88,6 @@ export default async function({ readConfigFile }) { '--xpack.encryptedSavedObjects.encryptionKey="DkdXazszSCYexXqz4YktBGHCRkV6hyNK"', '--telemetry.banner=false', '--timelion.ui.enabled=true', - '--xpack.endpoint.enabled=true', - '--xpack.ingestManager.enabled=true', - '--xpack.ingestManager.fleet.enabled=true', ], }, uiSettings: { diff --git a/x-pack/test/functional/apps/endpoint/alerts.ts b/x-pack/test/functional_endpoint/apps/endpoint/alerts.ts similarity index 97% rename from x-pack/test/functional/apps/endpoint/alerts.ts rename to x-pack/test/functional_endpoint/apps/endpoint/alerts.ts index 6a5a5cb5c32ad..44add06cf123b 100644 --- a/x-pack/test/functional/apps/endpoint/alerts.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/alerts.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; export default function({ getPageObjects, getService }: FtrProviderContext) { const pageObjects = getPageObjects(['common', 'endpointAlerts']); diff --git a/x-pack/test/functional/apps/endpoint/feature_controls/endpoint_spaces.ts b/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/endpoint_spaces.ts similarity index 96% rename from x-pack/test/functional/apps/endpoint/feature_controls/endpoint_spaces.ts rename to x-pack/test/functional_endpoint/apps/endpoint/feature_controls/endpoint_spaces.ts index c543046031e9f..ba2e7a248a9dd 100644 --- a/x-pack/test/functional/apps/endpoint/feature_controls/endpoint_spaces.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/endpoint_spaces.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; +import { FtrProviderContext } from '../../../../functional/ftr_provider_context'; export default function({ getPageObjects, getService }: FtrProviderContext) { const pageObjects = getPageObjects(['common']); diff --git a/x-pack/test/functional/apps/endpoint/feature_controls/index.ts b/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/index.ts similarity index 84% rename from x-pack/test/functional/apps/endpoint/feature_controls/index.ts rename to x-pack/test/functional_endpoint/apps/endpoint/feature_controls/index.ts index 5f7e611fd966c..46fde1596b0cf 100644 --- a/x-pack/test/functional/apps/endpoint/feature_controls/index.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/index.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../../../ftr_provider_context'; +import { FtrProviderContext } from '../../../../functional/ftr_provider_context'; export default function({ loadTestFile }: FtrProviderContext) { describe('feature controls', function() { diff --git a/x-pack/test/functional/apps/endpoint/header_nav.ts b/x-pack/test/functional_endpoint/apps/endpoint/header_nav.ts similarity index 96% rename from x-pack/test/functional/apps/endpoint/header_nav.ts rename to x-pack/test/functional_endpoint/apps/endpoint/header_nav.ts index c2c4068212484..b92e790f855a3 100644 --- a/x-pack/test/functional/apps/endpoint/header_nav.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/header_nav.ts @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { const pageObjects = getPageObjects(['common', 'endpoint']); diff --git a/x-pack/test/functional/apps/endpoint/host_list.ts b/x-pack/test/functional_endpoint/apps/endpoint/host_list.ts similarity index 98% rename from x-pack/test/functional/apps/endpoint/host_list.ts rename to x-pack/test/functional_endpoint/apps/endpoint/host_list.ts index 2e204775808c9..dc0ffea88a808 100644 --- a/x-pack/test/functional/apps/endpoint/host_list.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/host_list.ts @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { const pageObjects = getPageObjects(['common', 'endpoint', 'header']); diff --git a/x-pack/test/functional/apps/endpoint/index.ts b/x-pack/test/functional_endpoint/apps/endpoint/index.ts similarity index 89% rename from x-pack/test/functional/apps/endpoint/index.ts rename to x-pack/test/functional_endpoint/apps/endpoint/index.ts index 3e70a1cc67670..cec425d8599f7 100644 --- a/x-pack/test/functional/apps/endpoint/index.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/index.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; export default function({ loadTestFile }: FtrProviderContext) { describe('endpoint', function() { diff --git a/x-pack/test/functional/apps/endpoint/landing_page.ts b/x-pack/test/functional_endpoint/apps/endpoint/landing_page.ts similarity index 92% rename from x-pack/test/functional/apps/endpoint/landing_page.ts rename to x-pack/test/functional_endpoint/apps/endpoint/landing_page.ts index b4da4631aa60b..50ba161a85348 100644 --- a/x-pack/test/functional/apps/endpoint/landing_page.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/landing_page.ts @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { const pageObjects = getPageObjects(['common', 'endpoint']); diff --git a/x-pack/test/functional/apps/endpoint/policy_list.ts b/x-pack/test/functional_endpoint/apps/endpoint/policy_list.ts similarity index 95% rename from x-pack/test/functional/apps/endpoint/policy_list.ts rename to x-pack/test/functional_endpoint/apps/endpoint/policy_list.ts index c54eafdd8b787..acaaa02b3b30f 100644 --- a/x-pack/test/functional/apps/endpoint/policy_list.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/policy_list.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; export default function({ getPageObjects, getService }: FtrProviderContext) { const pageObjects = getPageObjects(['common', 'endpoint']); diff --git a/x-pack/test/functional_endpoint/config.ts b/x-pack/test/functional_endpoint/config.ts new file mode 100644 index 0000000000000..08cebfe026280 --- /dev/null +++ b/x-pack/test/functional_endpoint/config.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { resolve } from 'path'; +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; + +export default async function({ readConfigFile }: FtrConfigProviderContext) { + const xpackFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js')); + + return { + ...xpackFunctionalConfig.getAll(), + testFiles: [resolve(__dirname, './apps/endpoint')], + kbnTestServer: { + ...xpackFunctionalConfig.get('kbnTestServer'), + serverArgs: [ + ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'), + '--xpack.endpoint.enabled=true', + '--xpack.ingestManager.enabled=true', + '--xpack.ingestManager.fleet.enabled=true', + ], + }, + }; +} diff --git a/x-pack/test/functional_endpoint_ingest_failure/config.ts b/x-pack/test/functional_endpoint_ingest_failure/config.ts index 2d825a5a423a8..2207c842af656 100644 --- a/x-pack/test/functional_endpoint_ingest_failure/config.ts +++ b/x-pack/test/functional_endpoint_ingest_failure/config.ts @@ -17,6 +17,9 @@ export default async function({ readConfigFile }: FtrConfigProviderContext) { ...xpackFunctionalConfig.get('kbnTestServer'), serverArgs: [ ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'), + '--xpack.endpoint.enabled=true', + '--xpack.ingestManager.enabled=true', + '--xpack.ingestManager.fleet.enabled=true', // use a bogus port so the ingest manager setup will fail '--xpack.ingestManager.epm.registryUrl=http://127.0.0.1:12345', ], From b82762b7339642eae844c23c6e7f17450832e8ce Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 11:38:42 -0400 Subject: [PATCH 03/27] Removing page objects and other endpoint fields from base functional --- x-pack/test/functional/config.js | 4 ---- x-pack/test/functional/page_objects/index.ts | 4 ---- .../functional_endpoint/apps/endpoint/alerts.ts | 2 +- .../apps/endpoint/header_nav.ts | 2 +- .../apps/endpoint/host_list.ts | 2 +- .../functional_endpoint/apps/endpoint/index.ts | 2 +- .../apps/endpoint/landing_page.ts | 2 +- .../apps/endpoint/policy_list.ts | 2 +- x-pack/test/functional_endpoint/config.ts | 8 ++++++++ .../functional_endpoint/ftr_provider_context.d.ts | 12 ++++++++++++ .../page_objects/endpoint_alerts_page.ts | 0 .../page_objects/endpoint_page.ts | 0 .../functional_endpoint/page_objects/index.ts | 15 +++++++++++++++ .../apps/endpoint/index.ts | 2 +- .../apps/endpoint/landing_page.ts | 4 ++-- .../functional_endpoint_ingest_failure/config.ts | 7 +++---- .../ftr_provider_context.d.ts | 12 ++++++++++++ 17 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 x-pack/test/functional_endpoint/ftr_provider_context.d.ts rename x-pack/test/{functional => functional_endpoint}/page_objects/endpoint_alerts_page.ts (100%) rename x-pack/test/{functional => functional_endpoint}/page_objects/endpoint_page.ts (100%) create mode 100644 x-pack/test/functional_endpoint/page_objects/index.ts create mode 100644 x-pack/test/functional_endpoint_ingest_failure/ftr_provider_context.d.ts diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index b45577da9f282..bc9a67da731cc 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -57,7 +57,6 @@ export default async function({ readConfigFile }) { resolve(__dirname, './apps/cross_cluster_replication'), resolve(__dirname, './apps/remote_clusters'), resolve(__dirname, './apps/transform'), - resolve(__dirname, './apps/endpoint'), // This license_management file must be last because it is destructive. resolve(__dirname, './apps/license_management'), ], @@ -198,9 +197,6 @@ export default async function({ readConfigFile }) { pathname: '/app/kibana/', hash: '/management/elasticsearch/transform', }, - endpoint: { - pathname: '/app/endpoint', - }, }, // choose where esArchiver should load archives from diff --git a/x-pack/test/functional/page_objects/index.ts b/x-pack/test/functional/page_objects/index.ts index 07c5719ae53c5..782d57adea770 100644 --- a/x-pack/test/functional/page_objects/index.ts +++ b/x-pack/test/functional/page_objects/index.ts @@ -46,8 +46,6 @@ import { LensPageProvider } from './lens_page'; import { InfraMetricExplorerProvider } from './infra_metric_explorer'; import { RoleMappingsPageProvider } from './role_mappings_page'; import { SpaceSelectorPageProvider } from './space_selector_page'; -import { EndpointPageProvider } from './endpoint_page'; -import { EndpointAlertsPageProvider } from './endpoint_alerts_page'; // just like services, PageObjects are defined as a map of // names to Providers. Merge in Kibana's or pick specific ones @@ -81,6 +79,4 @@ export const pageObjects = { copySavedObjectsToSpace: CopySavedObjectsToSpacePageProvider, lens: LensPageProvider, roleMappings: RoleMappingsPageProvider, - endpoint: EndpointPageProvider, - endpointAlerts: EndpointAlertsPageProvider, }; diff --git a/x-pack/test/functional_endpoint/apps/endpoint/alerts.ts b/x-pack/test/functional_endpoint/apps/endpoint/alerts.ts index 44add06cf123b..6a5a5cb5c32ad 100644 --- a/x-pack/test/functional_endpoint/apps/endpoint/alerts.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/alerts.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function({ getPageObjects, getService }: FtrProviderContext) { const pageObjects = getPageObjects(['common', 'endpointAlerts']); diff --git a/x-pack/test/functional_endpoint/apps/endpoint/header_nav.ts b/x-pack/test/functional_endpoint/apps/endpoint/header_nav.ts index b92e790f855a3..c2c4068212484 100644 --- a/x-pack/test/functional_endpoint/apps/endpoint/header_nav.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/header_nav.ts @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { const pageObjects = getPageObjects(['common', 'endpoint']); diff --git a/x-pack/test/functional_endpoint/apps/endpoint/host_list.ts b/x-pack/test/functional_endpoint/apps/endpoint/host_list.ts index dc0ffea88a808..2e204775808c9 100644 --- a/x-pack/test/functional_endpoint/apps/endpoint/host_list.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/host_list.ts @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { const pageObjects = getPageObjects(['common', 'endpoint', 'header']); diff --git a/x-pack/test/functional_endpoint/apps/endpoint/index.ts b/x-pack/test/functional_endpoint/apps/endpoint/index.ts index cec425d8599f7..3e70a1cc67670 100644 --- a/x-pack/test/functional_endpoint/apps/endpoint/index.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/index.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function({ loadTestFile }: FtrProviderContext) { describe('endpoint', function() { diff --git a/x-pack/test/functional_endpoint/apps/endpoint/landing_page.ts b/x-pack/test/functional_endpoint/apps/endpoint/landing_page.ts index 50ba161a85348..b4da4631aa60b 100644 --- a/x-pack/test/functional_endpoint/apps/endpoint/landing_page.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/landing_page.ts @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { const pageObjects = getPageObjects(['common', 'endpoint']); diff --git a/x-pack/test/functional_endpoint/apps/endpoint/policy_list.ts b/x-pack/test/functional_endpoint/apps/endpoint/policy_list.ts index acaaa02b3b30f..c54eafdd8b787 100644 --- a/x-pack/test/functional_endpoint/apps/endpoint/policy_list.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/policy_list.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function({ getPageObjects, getService }: FtrProviderContext) { const pageObjects = getPageObjects(['common', 'endpoint']); diff --git a/x-pack/test/functional_endpoint/config.ts b/x-pack/test/functional_endpoint/config.ts index 08cebfe026280..b44df594b65bf 100644 --- a/x-pack/test/functional_endpoint/config.ts +++ b/x-pack/test/functional_endpoint/config.ts @@ -6,13 +6,21 @@ import { resolve } from 'path'; import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; +import { pageObjects } from './page_objects'; export default async function({ readConfigFile }: FtrConfigProviderContext) { const xpackFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js')); return { ...xpackFunctionalConfig.getAll(), + pageObjects, testFiles: [resolve(__dirname, './apps/endpoint')], + apps: { + ...xpackFunctionalConfig.get('apps'), + endpoint: { + pathname: '/app/endpoint', + }, + }, kbnTestServer: { ...xpackFunctionalConfig.get('kbnTestServer'), serverArgs: [ diff --git a/x-pack/test/functional_endpoint/ftr_provider_context.d.ts b/x-pack/test/functional_endpoint/ftr_provider_context.d.ts new file mode 100644 index 0000000000000..21ab5d5a4e554 --- /dev/null +++ b/x-pack/test/functional_endpoint/ftr_provider_context.d.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GenericFtrProviderContext } from '@kbn/test/types/ftr'; + +import { pageObjects } from './page_objects'; +import { services } from '../functional/services'; + +export type FtrProviderContext = GenericFtrProviderContext; diff --git a/x-pack/test/functional/page_objects/endpoint_alerts_page.ts b/x-pack/test/functional_endpoint/page_objects/endpoint_alerts_page.ts similarity index 100% rename from x-pack/test/functional/page_objects/endpoint_alerts_page.ts rename to x-pack/test/functional_endpoint/page_objects/endpoint_alerts_page.ts diff --git a/x-pack/test/functional/page_objects/endpoint_page.ts b/x-pack/test/functional_endpoint/page_objects/endpoint_page.ts similarity index 100% rename from x-pack/test/functional/page_objects/endpoint_page.ts rename to x-pack/test/functional_endpoint/page_objects/endpoint_page.ts diff --git a/x-pack/test/functional_endpoint/page_objects/index.ts b/x-pack/test/functional_endpoint/page_objects/index.ts new file mode 100644 index 0000000000000..8138ce2eeccb3 --- /dev/null +++ b/x-pack/test/functional_endpoint/page_objects/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pageObjects as xpackFunctionalPageObjects } from '../../functional/page_objects'; +import { EndpointPageProvider } from './endpoint_page'; +import { EndpointAlertsPageProvider } from './endpoint_alerts_page'; + +export const pageObjects = { + ...xpackFunctionalPageObjects, + endpoint: EndpointPageProvider, + endpointAlerts: EndpointAlertsPageProvider, +}; diff --git a/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/index.ts b/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/index.ts index 068880d3a8433..ae35f3e525461 100644 --- a/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/index.ts +++ b/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/index.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function({ loadTestFile }: FtrProviderContext) { describe('endpoint when the ingest manager fails to setup correctly', function() { diff --git a/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/landing_page.ts b/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/landing_page.ts index c5f3016c15394..d29250ca3bed4 100644 --- a/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/landing_page.ts +++ b/x-pack/test/functional_endpoint_ingest_failure/apps/endpoint/landing_page.ts @@ -4,11 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { describe('home page', function() { - const pageObjects = getPageObjects(['common', 'endpoint']); + const pageObjects = getPageObjects(['common']); const testSubjects = getService('testSubjects'); before(async () => { diff --git a/x-pack/test/functional_endpoint_ingest_failure/config.ts b/x-pack/test/functional_endpoint_ingest_failure/config.ts index 2207c842af656..347cd0192bbdb 100644 --- a/x-pack/test/functional_endpoint_ingest_failure/config.ts +++ b/x-pack/test/functional_endpoint_ingest_failure/config.ts @@ -8,7 +8,9 @@ import { resolve } from 'path'; import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; export default async function({ readConfigFile }: FtrConfigProviderContext) { - const xpackFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js')); + const xpackFunctionalConfig = await readConfigFile( + require.resolve('../functional_endpoint/config.ts') + ); return { ...xpackFunctionalConfig.getAll(), @@ -17,9 +19,6 @@ export default async function({ readConfigFile }: FtrConfigProviderContext) { ...xpackFunctionalConfig.get('kbnTestServer'), serverArgs: [ ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'), - '--xpack.endpoint.enabled=true', - '--xpack.ingestManager.enabled=true', - '--xpack.ingestManager.fleet.enabled=true', // use a bogus port so the ingest manager setup will fail '--xpack.ingestManager.epm.registryUrl=http://127.0.0.1:12345', ], diff --git a/x-pack/test/functional_endpoint_ingest_failure/ftr_provider_context.d.ts b/x-pack/test/functional_endpoint_ingest_failure/ftr_provider_context.d.ts new file mode 100644 index 0000000000000..0e4b47471d419 --- /dev/null +++ b/x-pack/test/functional_endpoint_ingest_failure/ftr_provider_context.d.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GenericFtrProviderContext } from '@kbn/test/types/ftr'; + +import { pageObjects } from '../functional_endpoint/page_objects'; +import { services } from '../functional/services'; + +export type FtrProviderContext = GenericFtrProviderContext; From cb45dfe6fde4bffa3f985e2374e552686b70f30d Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 11:41:50 -0400 Subject: [PATCH 04/27] Updating code owners with new functional location --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 635f44c3596e2..e707250ff3261 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -202,7 +202,7 @@ # Endpoint /x-pack/plugins/endpoint/ @elastic/endpoint-app-team /x-pack/test/api_integration/apis/endpoint/ @elastic/endpoint-app-team -/x-pack/test/functional/apps/endpoint/ @elastic/endpoint-app-team +/x-pack/test/functional_endpoint/ @elastic/endpoint-app-team /x-pack/test/functional_endpoint_ingest_failure/ @elastic/endpoint-app-team /x-pack/test/functional/es_archives/endpoint/ @elastic/endpoint-app-team From 9e4f6e3fb32fd8b3d0590cfccd23055dcf663415 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 13:00:43 -0400 Subject: [PATCH 05/27] Adding index pattern functionality --- .../endpoint/common/schema/index_pattern.ts | 12 + x-pack/plugins/endpoint/common/types.ts | 25 +- .../endpoint/store/alerts/middleware.ts | 10 +- .../endpoint/scripts/resolver_generator.ts | 4 +- .../plugins/endpoint/server/index_pattern.ts | 36 +++ x-pack/plugins/endpoint/server/plugin.test.ts | 39 ++- x-pack/plugins/endpoint/server/plugin.ts | 9 + .../server/routes/alerts/alerts.test.ts | 3 + .../server/routes/alerts/details/handlers.ts | 14 +- .../routes/alerts/details/lib/pagination.ts | 6 +- .../server/routes/alerts/lib/index.ts | 10 +- .../server/routes/alerts/list/handlers.ts | 12 +- .../server/routes/alerts/list/lib/index.ts | 2 + .../endpoint/server/routes/index_pattern.ts | 49 +++ .../endpoint/server/routes/metadata/index.ts | 26 +- .../server/routes/metadata/metadata.test.ts | 66 ++-- .../routes/metadata/query_builders.test.ts | 47 +-- .../server/routes/metadata/query_builders.ts | 16 +- .../endpoint/server/routes/resolver.ts | 7 +- .../server/routes/resolver/children.ts | 13 +- .../server/routes/resolver/lifecycle.ts | 12 +- .../server/routes/resolver/queries/base.ts | 3 +- .../routes/resolver/queries/children.test.ts | 16 +- .../routes/resolver/queries/lifecycle.test.ts | 9 +- .../resolver/queries/related_events.test.ts | 15 +- .../server/routes/resolver/related_events.ts | 11 +- .../server/test_data/all_metadata_data.json | 284 +++++++++--------- x-pack/plugins/endpoint/server/types.ts | 2 + .../ingest_manager/common/types/models/epm.ts | 7 +- .../plugins/ingest_manager/server/plugin.ts | 23 +- .../server/routes/epm/handlers.ts | 2 +- .../ingest_manager/server/saved_objects.ts | 14 +- .../epm/elasticsearch/template/template.ts | 16 + .../server/services/epm/packages/install.ts | 44 ++- .../server/services/epm/packages/remove.ts | 6 +- .../ingest_manager/server/services/index.ts | 1 + .../server/services/index_pattern.ts | 39 +++ .../ingest_manager/server/types/index.tsx | 1 + .../api_integration/apis/endpoint/index.ts | 1 + .../apis/endpoint/index_pattern.ts | 36 +++ .../api_integration/apis/endpoint/metadata.ts | 2 +- .../api_integration/apis/endpoint/resolver.ts | 4 +- 42 files changed, 671 insertions(+), 283 deletions(-) create mode 100644 x-pack/plugins/endpoint/common/schema/index_pattern.ts create mode 100644 x-pack/plugins/endpoint/server/index_pattern.ts create mode 100644 x-pack/plugins/endpoint/server/routes/index_pattern.ts create mode 100644 x-pack/plugins/ingest_manager/server/services/index_pattern.ts create mode 100644 x-pack/test/api_integration/apis/endpoint/index_pattern.ts diff --git a/x-pack/plugins/endpoint/common/schema/index_pattern.ts b/x-pack/plugins/endpoint/common/schema/index_pattern.ts new file mode 100644 index 0000000000000..32144f820eca6 --- /dev/null +++ b/x-pack/plugins/endpoint/common/schema/index_pattern.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; + +export const indexPatternGetParamsSchema = schema.object({ datasetPath: schema.string() }); +export const indexPatternGetQueryParamsSchema = schema.object({ + version: schema.maybe(schema.string()), +}); diff --git a/x-pack/plugins/endpoint/common/types.ts b/x-pack/plugins/endpoint/common/types.ts index e8e1281a88925..bd402f82dbd44 100644 --- a/x-pack/plugins/endpoint/common/types.ts +++ b/x-pack/plugins/endpoint/common/types.ts @@ -7,6 +7,10 @@ import { SearchResponse } from 'elasticsearch'; import { TypeOf } from '@kbn/config-schema'; import { alertingIndexGetQuerySchema } from './schema/alert_index'; +import { + indexPatternGetParamsSchema, + indexPatternGetQueryParamsSchema, +} from './schema/index_pattern'; /** * A deep readonly type that will make all children of a given object readonly recursively @@ -30,9 +34,9 @@ export type Direction = 'asc' | 'desc'; export class EndpointAppConstants { static BASE_API_URL = '/api/endpoint'; - static ENDPOINT_INDEX_NAME = 'endpoint-agent*'; static ALERT_INDEX_NAME = 'events-endpoint-1'; - static EVENT_INDEX_NAME = 'events-endpoint-*'; + static EVENT_DATASET = 'events'; + static METADATA_DATASET = 'metadata'; static DEFAULT_TOTAL_HITS = 10000; /** * Legacy events are stored in indices with endgame-* prefix @@ -421,3 +425,20 @@ export type AlertingIndexGetQueryInput = KbnConfigSchemaInputTypeOf< * Result of the validated query params when handling alert index requests. */ export type AlertingIndexGetQueryResult = TypeOf; + +/** + * Result of the validated params when handling an index pattern request. + */ +export type IndexPatternGetParamsResult = TypeOf; + +/** + * Query params to pass to the index pattern API when fetching an index pattern. + */ +export type IndexPatternGetQueryParamsInput = KbnConfigSchemaInputTypeOf< + TypeOf +>; + +/** + * Result of the validated query params when handling an index pattern request. + */ +export type IndexPatternGetQueryParamsResult = TypeOf; diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts b/x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts index 2c6ebf52189f5..bd2c695d9360f 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts +++ b/x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts @@ -15,10 +15,14 @@ import { EndpointAppConstants } from '../../../../../common/types'; export const alertMiddlewareFactory: MiddlewareFactory = (coreStart, depsStart) => { async function fetchIndexPatterns(): Promise { const { indexPatterns } = depsStart.data; - const indexName = EndpointAppConstants.ALERT_INDEX_NAME; - const fields = await indexPatterns.getFieldsForWildcard({ pattern: indexName }); + const eventsPattern: { indexPattern: string } = await coreStart.http.get( + `/api/endpoint/index_pattern/${EndpointAppConstants.EVENT_DATASET}` + ); + const fields = await indexPatterns.getFieldsForWildcard({ + pattern: eventsPattern.indexPattern, + }); const indexPattern: IIndexPattern = { - title: indexName, + title: eventsPattern.indexPattern, fields, }; diff --git a/x-pack/plugins/endpoint/scripts/resolver_generator.ts b/x-pack/plugins/endpoint/scripts/resolver_generator.ts index 333846bde6ce4..d409b42cfa261 100644 --- a/x-pack/plugins/endpoint/scripts/resolver_generator.ts +++ b/x-pack/plugins/endpoint/scripts/resolver_generator.ts @@ -35,13 +35,13 @@ async function main() { eventIndex: { alias: 'ei', describe: 'index to store events in', - default: 'events-endpoint-1', + default: 'events-endpoint-default-1', type: 'string', }, metadataIndex: { alias: 'mi', describe: 'index to store host metadata in', - default: 'endpoint-agent-1', + default: 'metrics-endpoint-default-1', type: 'string', }, auth: { diff --git a/x-pack/plugins/endpoint/server/index_pattern.ts b/x-pack/plugins/endpoint/server/index_pattern.ts new file mode 100644 index 0000000000000..38d69aedf3e81 --- /dev/null +++ b/x-pack/plugins/endpoint/server/index_pattern.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SavedObjectsClientContract, Logger, LoggerFactory } from 'kibana/server'; +import { IndexPatternService } from '../../ingest_manager/server'; + +const endpointPackageName = 'endpoint'; + +export interface IndexPatternRetriever { + get(client: SavedObjectsClientContract, datasetPath: string, version?: string): Promise; +} + +export class IngestIndexPatternRetriever implements IndexPatternRetriever { + private readonly log: Logger; + constructor(private readonly service: IndexPatternService, loggerFactory: LoggerFactory) { + this.log = loggerFactory.get('index-pattern-retriever'); + } + + async get(client: SavedObjectsClientContract, datasetPath: string, version?: string) { + const pattern = await this.service.get(client, endpointPackageName, datasetPath, version); + + if (!pattern) { + const versionStr = version || 'none'; + this.log.warn( + `Failed to retrieve index pattern from ingest manager dataset: ${datasetPath} version: ${versionStr}` + ); + throw new Error( + `Unable to retrieve the index pattern for dataset: ${datasetPath} version: ${version}` + ); + } + + return pattern; + } +} diff --git a/x-pack/plugins/endpoint/server/plugin.test.ts b/x-pack/plugins/endpoint/server/plugin.test.ts index 7dd878d579043..fd308c11cdbf7 100644 --- a/x-pack/plugins/endpoint/server/plugin.test.ts +++ b/x-pack/plugins/endpoint/server/plugin.test.ts @@ -3,10 +3,44 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +/* eslint-disable max-classes-per-file */ +import { SavedObjectsClientContract } from 'kibana/server'; import { EndpointPlugin, EndpointPluginSetupDependencies } from './plugin'; import { coreMock } from '../../../../src/core/server/mocks'; import { PluginSetupContract } from '../../features/server'; +import { IndexPatternService } from '../../ingest_manager/server'; + +export const MetadataIndexPattern = 'metadata-endpoint-*'; + +export interface IndexPatternRetriever { + get(client: SavedObjectsClientContract, datasetPath: string, version?: string): Promise; +} + +export class FakeIndexPatternRetriever implements IndexPatternRetriever { + constructor(private readonly indexPattern: string) {} + + static buildMetadata() { + return new FakeIndexPatternRetriever(MetadataIndexPattern); + } + + async get(client: SavedObjectsClientContract, datasetPath: string, version?: string) { + return this.indexPattern; + } +} + +export class FakeIndexPatternService implements IndexPatternService { + constructor(private readonly indexPattern: string) {} + + async get( + savedObjectsClient: SavedObjectsClientContract, + pkgName: string, + datasetPath: string, + version?: string + ): Promise { + return this.indexPattern; + } +} describe('test endpoint plugin', () => { let plugin: EndpointPlugin; @@ -28,7 +62,10 @@ describe('test endpoint plugin', () => { getFeaturesUICapabilities: jest.fn(), registerLegacyAPI: jest.fn(), }; - mockedEndpointPluginSetupDependencies = { features: mockedPluginSetupContract }; + mockedEndpointPluginSetupDependencies = { + features: mockedPluginSetupContract, + ingestManager: { indexPatternService: new FakeIndexPatternService('') }, + }; }); it('test properly setup plugin', async () => { diff --git a/x-pack/plugins/endpoint/server/plugin.ts b/x-pack/plugins/endpoint/server/plugin.ts index d3a399124124f..e4e41045a6c10 100644 --- a/x-pack/plugins/endpoint/server/plugin.ts +++ b/x-pack/plugins/endpoint/server/plugin.ts @@ -6,12 +6,15 @@ import { Plugin, CoreSetup, PluginInitializerContext, Logger } from 'kibana/server'; import { first } from 'rxjs/operators'; import { PluginSetupContract as FeaturesPluginSetupContract } from '../../features/server'; +import { IngestManagerSetupContract } from '../../ingest_manager/server'; import { createConfig$, EndpointConfigType } from './config'; import { EndpointAppContext } from './types'; import { registerAlertRoutes } from './routes/alerts'; import { registerResolverRoutes } from './routes/resolver'; +import { registerIndexPatternRoute } from './routes/index_pattern'; import { registerEndpointRoutes } from './routes/metadata'; +import { IngestIndexPatternRetriever } from './index_pattern'; export type EndpointPluginStart = void; export type EndpointPluginSetup = void; @@ -19,6 +22,7 @@ export interface EndpointPluginStartDependencies {} // eslint-disable-line @type export interface EndpointPluginSetupDependencies { features: FeaturesPluginSetupContract; + ingestManager: IngestManagerSetupContract; } export class EndpointPlugin @@ -62,6 +66,10 @@ export class EndpointPlugin }, }); const endpointContext = { + indexPatternRetriever: new IngestIndexPatternRetriever( + plugins.ingestManager.indexPatternService, + this.initializerContext.logger + ), logFactory: this.initializerContext.logger, config: (): Promise => { return createConfig$(this.initializerContext) @@ -73,6 +81,7 @@ export class EndpointPlugin registerEndpointRoutes(router, endpointContext); registerResolverRoutes(router, endpointContext); registerAlertRoutes(router, endpointContext); + registerIndexPatternRoute(router, endpointContext); } public start() { diff --git a/x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts b/x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts index 5f5e4be4ecd0a..e245914360521 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts @@ -12,6 +12,7 @@ import { import { registerAlertRoutes } from './index'; import { EndpointConfigSchema } from '../../config'; import { alertingIndexGetQuerySchema } from '../../../common/schema/alert_index'; +import { FakeIndexPatternRetriever } from '../../plugin.test'; describe('test alerts route', () => { let routerMock: jest.Mocked; @@ -24,6 +25,8 @@ describe('test alerts route', () => { mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); registerAlertRoutes(routerMock, { + indexPatternRetriever: new FakeIndexPatternRetriever('events-endpoint-*'), + logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }); diff --git a/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts b/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts index 725e362f91ec7..8bcd0c60de6af 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts @@ -22,22 +22,32 @@ export const alertDetailsHandlerWrapper = function( try { const alertId = req.params.id; const response = (await ctx.core.elasticsearch.dataClient.callAsCurrentUser('get', { + // Remove the hard coded reference to an alert index here + // https://github.com/elastic/endpoint-app-team/issues/311 index: EndpointAppConstants.ALERT_INDEX_NAME, id: alertId, })) as GetResponse; + const indexPattern = await endpointAppContext.indexPatternRetriever.get( + ctx.core.savedObjects.client, + EndpointAppConstants.EVENT_DATASET + ); + const config = await endpointAppContext.config(); const pagination: AlertDetailsPagination = new AlertDetailsPagination( config, ctx, req.params, - response + response, + indexPattern ); - const currentHostInfo = await getHostData(ctx, response._source.host.id); + const currentHostInfo = await getHostData(ctx, response._source.host.id, indexPattern); return res.ok({ body: { + // base64 encode the index in the response + // https://github.com/elastic/endpoint-app-team/issues/311 id: response._id, ...response._source, state: { diff --git a/x-pack/plugins/endpoint/server/routes/alerts/details/lib/pagination.ts b/x-pack/plugins/endpoint/server/routes/alerts/details/lib/pagination.ts index 446d61080e650..d482da03872c6 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/details/lib/pagination.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/details/lib/pagination.ts @@ -29,7 +29,8 @@ export class AlertDetailsPagination extends Pagination< config: EndpointConfigType, requestContext: RequestHandlerContext, state: AlertDetailsRequestParams, - data: GetResponse + data: GetResponse, + private readonly indexPattern: string ) { super(config, requestContext, state, data); } @@ -54,7 +55,8 @@ export class AlertDetailsPagination extends Pagination< const response = await searchESForAlerts( this.requestContext.core.elasticsearch.dataClient, - reqData + reqData, + this.indexPattern ); return response; } diff --git a/x-pack/plugins/endpoint/server/routes/alerts/lib/index.ts b/x-pack/plugins/endpoint/server/routes/alerts/lib/index.ts index dc1ce767a715b..74db24c85eab5 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/lib/index.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/lib/index.ts @@ -97,7 +97,8 @@ function buildSort(query: AlertSearchQuery): AlertSort { * Builds a request body for Elasticsearch, given a set of query params. **/ const buildAlertSearchQuery = async ( - query: AlertSearchQuery + query: AlertSearchQuery, + indexPattern: string ): Promise => { let totalHitsMin: number = EndpointAppConstants.DEFAULT_TOTAL_HITS; @@ -125,7 +126,7 @@ const buildAlertSearchQuery = async ( const reqWrapper: AlertSearchRequestWrapper = { size: query.pageSize, - index: EndpointAppConstants.ALERT_INDEX_NAME, + index: indexPattern, body: reqBody, }; @@ -141,9 +142,10 @@ const buildAlertSearchQuery = async ( **/ export const searchESForAlerts = async ( dataClient: IScopedClusterClient, - query: AlertSearchQuery + query: AlertSearchQuery, + indexPattern: string ): Promise> => { - const reqWrapper = await buildAlertSearchQuery(query); + const reqWrapper = await buildAlertSearchQuery(query, indexPattern); const response = (await dataClient.callAsCurrentUser('search', reqWrapper)) as SearchResponse< AlertEvent >; diff --git a/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts b/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts index 93ec8d7aa3e67..8441034c35edf 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts @@ -7,7 +7,7 @@ import { RequestHandler } from 'kibana/server'; import { EndpointAppContext } from '../../../types'; import { searchESForAlerts } from '../lib'; import { getRequestData, mapToAlertResultList } from './lib'; -import { AlertingIndexGetQueryResult } from '../../../../common/types'; +import { AlertingIndexGetQueryResult, EndpointAppConstants } from '../../../../common/types'; export const alertListHandlerWrapper = function( endpointAppContext: EndpointAppContext @@ -18,8 +18,16 @@ export const alertListHandlerWrapper = function( res ) => { try { + const indexPattern = await endpointAppContext.indexPatternRetriever.get( + ctx.core.savedObjects.client, + EndpointAppConstants.EVENT_DATASET + ); const reqData = await getRequestData(req, endpointAppContext); - const response = await searchESForAlerts(ctx.core.elasticsearch.dataClient, reqData); + const response = await searchESForAlerts( + ctx.core.elasticsearch.dataClient, + reqData, + indexPattern + ); const mappedBody = await mapToAlertResultList(ctx, endpointAppContext, reqData, response); return res.ok({ body: mappedBody }); } catch (err) { diff --git a/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts b/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts index 95c8e4662cfce..a1b6e8eac7d04 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts @@ -108,6 +108,8 @@ export async function mapToAlertResultList( function mapHit(entry: AlertHits[0]): AlertData { return { + // base64 encode the alert index in the id field here + // https://github.com/elastic/endpoint-app-team/issues/311 id: entry._id, ...entry._source, prev: null, diff --git a/x-pack/plugins/endpoint/server/routes/index_pattern.ts b/x-pack/plugins/endpoint/server/routes/index_pattern.ts new file mode 100644 index 0000000000000..79494ece5e956 --- /dev/null +++ b/x-pack/plugins/endpoint/server/routes/index_pattern.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter, Logger, RequestHandler } from 'kibana/server'; +import { EndpointAppContext } from '../types'; +import { IndexPatternGetParamsResult, IndexPatternGetQueryParamsResult } from '../../common/types'; +import { + indexPatternGetParamsSchema, + indexPatternGetQueryParamsSchema, +} from '../../common/schema/index_pattern'; +import { IndexPatternRetriever } from '../index_pattern'; + +function handleIndexPattern( + log: Logger, + indexRetriever: IndexPatternRetriever +): RequestHandler { + return async (context, req, res) => { + try { + return res.ok({ + body: { + indexPattern: await indexRetriever.get( + context.core.savedObjects.client, + req.params.datasetPath, + req.query.version + ), + }, + }); + } catch (error) { + log.warn(error); + return res.notFound({ body: error }); + } + }; +} + +export function registerIndexPatternRoute(router: IRouter, endpointAppContext: EndpointAppContext) { + const log = endpointAppContext.logFactory.get('index_pattern'); + + router.get( + { + path: '/api/endpoint/index_pattern/{datasetPath}', + validate: { params: indexPatternGetParamsSchema, query: indexPatternGetQueryParamsSchema }, + options: { authRequired: true }, + }, + handleIndexPattern(log, endpointAppContext.indexPatternRetriever) + ); +} diff --git a/x-pack/plugins/endpoint/server/routes/metadata/index.ts b/x-pack/plugins/endpoint/server/routes/metadata/index.ts index ef01db9af98c4..65deec95aa7d5 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/index.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/index.ts @@ -8,9 +8,9 @@ import { IRouter, RequestHandlerContext } from 'kibana/server'; import { SearchResponse } from 'elasticsearch'; import { schema } from '@kbn/config-schema'; -import { HostMetadata, HostResultList } from '../../../common/types'; +import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders'; +import { HostMetadata, HostResultList, EndpointAppConstants } from '../../../common/types'; import { EndpointAppContext } from '../../types'; -import { getESQueryHostMetadataByID, kibanaRequestToMetadataListESQuery } from './query_builders'; interface HitSource { _source: HostMetadata; @@ -50,7 +50,15 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp }, async (context, req, res) => { try { - const queryParams = await kibanaRequestToMetadataListESQuery(req, endpointAppContext); + const index = await endpointAppContext.indexPatternRetriever.get( + context.core.savedObjects.client, + EndpointAppConstants.METADATA_DATASET + ); + const queryParams = await kibanaRequestToMetadataListESQuery( + req, + endpointAppContext, + index + ); const response = (await context.core.elasticsearch.dataClient.callAsCurrentUser( 'search', queryParams @@ -72,7 +80,12 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp }, async (context, req, res) => { try { - const doc = await getHostData(context, req.params.id); + const index = await endpointAppContext.indexPatternRetriever.get( + context.core.savedObjects.client, + EndpointAppConstants.METADATA_DATASET + ); + + const doc = await getHostData(context, req.params.id, index); if (doc) { return res.ok({ body: doc }); } @@ -86,9 +99,10 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp export async function getHostData( context: RequestHandlerContext, - id: string + id: string, + index: string ): Promise { - const query = getESQueryHostMetadataByID(id); + const query = getESQueryHostMetadataByID(id, index); const response = (await context.core.elasticsearch.dataClient.callAsCurrentUser( 'search', query diff --git a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts index e0fd11e737e7d..8d2415ad22035 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts @@ -11,24 +11,28 @@ import { RequestHandler, RequestHandlerContext, RouteConfig, + SavedObjectsClientContract, } from 'kibana/server'; import { elasticsearchServiceMock, httpServerMock, httpServiceMock, loggingServiceMock, + savedObjectsClientMock, } from '../../../../../../src/core/server/mocks'; import { HostMetadata, HostResultList } from '../../../common/types'; import { SearchResponse } from 'elasticsearch'; -import { EndpointConfigSchema } from '../../config'; -import * as data from '../../test_data/all_metadata_data.json'; import { registerEndpointRoutes } from './index'; +import { EndpointConfigSchema } from '../../config'; +import { FakeIndexPatternRetriever } from '../../plugin.test'; +import * as data from '../test_data/all_metadata_data.json'; describe('test endpoint route', () => { let routerMock: jest.Mocked; let mockResponse: jest.Mocked; let mockClusterClient: jest.Mocked; let mockScopedClient: jest.Mocked; + let mockSavedObjectClient: jest.Mocked; let routeHandler: RequestHandler; let routeConfig: RouteConfig; @@ -37,15 +41,33 @@ describe('test endpoint route', () => { IClusterClient >; mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); + mockSavedObjectClient = savedObjectsClientMock.create(); mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); mockResponse = httpServerMock.createResponseFactory(); registerEndpointRoutes(routerMock, { + indexPatternRetriever: FakeIndexPatternRetriever.buildMetadata(), logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }); }); + function createRouteHandlerContext( + dataClient: jest.Mocked, + savedObjectsClient: jest.Mocked + ) { + return ({ + core: { + elasticsearch: { + dataClient, + }, + savedObjects: { + client: savedObjectsClient, + }, + }, + } as unknown) as RequestHandlerContext; + } + it('test find the latest of all endpoints', async () => { const mockRequest = httpServerMock.createKibanaRequest({}); @@ -58,13 +80,7 @@ describe('test endpoint route', () => { )!; await routeHandler( - ({ - core: { - elasticsearch: { - dataClient: mockScopedClient, - }, - }, - } as unknown) as RequestHandlerContext, + createRouteHandlerContext(mockScopedClient, mockSavedObjectClient), mockRequest, mockResponse ); @@ -100,13 +116,7 @@ describe('test endpoint route', () => { )!; await routeHandler( - ({ - core: { - elasticsearch: { - dataClient: mockScopedClient, - }, - }, - } as unknown) as RequestHandlerContext, + createRouteHandlerContext(mockScopedClient, mockSavedObjectClient), mockRequest, mockResponse ); @@ -147,13 +157,7 @@ describe('test endpoint route', () => { )!; await routeHandler( - ({ - core: { - elasticsearch: { - dataClient: mockScopedClient, - }, - }, - } as unknown) as RequestHandlerContext, + createRouteHandlerContext(mockScopedClient, mockSavedObjectClient), mockRequest, mockResponse ); @@ -212,13 +216,7 @@ describe('test endpoint route', () => { )!; await routeHandler( - ({ - core: { - elasticsearch: { - dataClient: mockScopedClient, - }, - }, - } as unknown) as RequestHandlerContext, + createRouteHandlerContext(mockScopedClient, mockSavedObjectClient), mockRequest, mockResponse ); @@ -243,13 +241,7 @@ describe('test endpoint route', () => { )!; await routeHandler( - ({ - core: { - elasticsearch: { - dataClient: mockScopedClient, - }, - }, - } as unknown) as RequestHandlerContext, + createRouteHandlerContext(mockScopedClient, mockSavedObjectClient), mockRequest, mockResponse ); diff --git a/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts b/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts index 2514d5aa85811..cb17b7bc000c9 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts @@ -6,7 +6,7 @@ import { httpServerMock, loggingServiceMock } from '../../../../../../src/core/server/mocks'; import { EndpointConfigSchema } from '../../config'; import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders'; -import { EndpointAppConstants } from '../../../common/types'; +import { MetadataIndexPattern, FakeIndexPatternRetriever } from '../../plugin.test'; describe('query builder', () => { describe('MetadataListESQuery', () => { @@ -14,17 +14,22 @@ describe('query builder', () => { const mockRequest = httpServerMock.createKibanaRequest({ body: {}, }); - const query = await kibanaRequestToMetadataListESQuery(mockRequest, { - logFactory: loggingServiceMock.create(), - config: () => Promise.resolve(EndpointConfigSchema.validate({})), - }); + const query = await kibanaRequestToMetadataListESQuery( + mockRequest, + { + indexPatternRetriever: FakeIndexPatternRetriever.buildMetadata(), + logFactory: loggingServiceMock.create(), + config: () => Promise.resolve(EndpointConfigSchema.validate({})), + }, + MetadataIndexPattern + ); expect(query).toEqual({ body: { query: { match_all: {}, }, collapse: { - field: 'host.id.keyword', + field: 'host.id', inner_hits: { name: 'most_recent', size: 1, @@ -34,7 +39,7 @@ describe('query builder', () => { aggs: { total: { cardinality: { - field: 'host.id.keyword', + field: 'host.id', }, }, }, @@ -48,7 +53,7 @@ describe('query builder', () => { }, from: 0, size: 10, - index: EndpointAppConstants.ENDPOINT_INDEX_NAME, + index: MetadataIndexPattern, } as Record); }); }); @@ -60,10 +65,15 @@ describe('query builder', () => { filter: 'not host.ip:10.140.73.246', }, }); - const query = await kibanaRequestToMetadataListESQuery(mockRequest, { - logFactory: loggingServiceMock.create(), - config: () => Promise.resolve(EndpointConfigSchema.validate({})), - }); + const query = await kibanaRequestToMetadataListESQuery( + mockRequest, + { + indexPatternRetriever: FakeIndexPatternRetriever.buildMetadata(), + logFactory: loggingServiceMock.create(), + config: () => Promise.resolve(EndpointConfigSchema.validate({})), + }, + MetadataIndexPattern + ); expect(query).toEqual({ body: { query: { @@ -83,7 +93,7 @@ describe('query builder', () => { }, }, collapse: { - field: 'host.id.keyword', + field: 'host.id', inner_hits: { name: 'most_recent', size: 1, @@ -93,7 +103,7 @@ describe('query builder', () => { aggs: { total: { cardinality: { - field: 'host.id.keyword', + field: 'host.id', }, }, }, @@ -107,7 +117,7 @@ describe('query builder', () => { }, from: 0, size: 10, - index: EndpointAppConstants.ENDPOINT_INDEX_NAME, + index: MetadataIndexPattern, } as Record); }); }); @@ -115,14 +125,15 @@ describe('query builder', () => { describe('MetadataGetQuery', () => { it('searches for the correct ID', () => { const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899'; - const query = getESQueryHostMetadataByID(mockID); + const query = getESQueryHostMetadataByID(mockID, MetadataIndexPattern); + expect(query).toEqual({ body: { - query: { match: { 'host.id.keyword': mockID } }, + query: { match: { 'host.id': mockID } }, sort: [{ 'event.created': { order: 'desc' } }], size: 1, }, - index: EndpointAppConstants.ENDPOINT_INDEX_NAME, + index: MetadataIndexPattern, }); }); }); diff --git a/x-pack/plugins/endpoint/server/routes/metadata/query_builders.ts b/x-pack/plugins/endpoint/server/routes/metadata/query_builders.ts index bd07604fe9ad2..abcc293985f9f 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/query_builders.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/query_builders.ts @@ -6,18 +6,18 @@ import { KibanaRequest } from 'kibana/server'; import { esKuery } from '../../../../../../src/plugins/data/server'; import { EndpointAppContext } from '../../types'; -import { EndpointAppConstants } from '../../../common/types'; export const kibanaRequestToMetadataListESQuery = async ( request: KibanaRequest, - endpointAppContext: EndpointAppContext + endpointAppContext: EndpointAppContext, + index: string ): Promise> => { const pagingProperties = await getPagingProperties(request, endpointAppContext); return { body: { query: buildQueryBody(request), collapse: { - field: 'host.id.keyword', + field: 'host.id', inner_hits: { name: 'most_recent', size: 1, @@ -27,7 +27,7 @@ export const kibanaRequestToMetadataListESQuery = async ( aggs: { total: { cardinality: { - field: 'host.id.keyword', + field: 'host.id', }, }, }, @@ -41,7 +41,7 @@ export const kibanaRequestToMetadataListESQuery = async ( }, from: pagingProperties.pageIndex * pagingProperties.pageSize, size: pagingProperties.pageSize, - index: EndpointAppConstants.ENDPOINT_INDEX_NAME, + index, }; }; @@ -74,12 +74,12 @@ function buildQueryBody(request: KibanaRequest): Record { return async (context, req, res) => { const { @@ -54,10 +57,14 @@ export function handleChildren( } = req; try { const pagination = getPaginationParams(limit, after); + const indexPattern = await indexRetriever.get( + context.core.savedObjects.client, + EndpointAppConstants.EVENT_DATASET + ); const client = context.core.elasticsearch.dataClient; - const childrenQuery = new ChildrenQuery(legacyEndpointID, pagination); - const lifecycleQuery = new LifecycleQuery(legacyEndpointID); + const childrenQuery = new ChildrenQuery(indexPattern, legacyEndpointID, pagination); + const lifecycleQuery = new LifecycleQuery(indexPattern, legacyEndpointID); // Retrieve the related child process events for a given process const { total, results: events, nextCursor } = await childrenQuery.search(client, id); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts b/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts index 9895344174014..0da1cfdb32761 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts @@ -10,6 +10,8 @@ import { RequestHandler, Logger } from 'kibana/server'; import { extractParentEntityID } from './utils/normalize'; import { LifecycleQuery } from './queries/lifecycle'; import { ResolverEvent } from '../../../common/types'; +import { EndpointAppConstants } from '../../../common/types'; +import { IndexPatternRetriever } from '../../index_pattern'; interface LifecycleQueryParams { ancestors: number; @@ -46,7 +48,8 @@ function getParentEntityID(results: ResolverEvent[]) { } export function handleLifecycle( - log: Logger + log: Logger, + indexRetriever: IndexPatternRetriever ): RequestHandler { return async (context, req, res) => { const { @@ -56,8 +59,11 @@ export function handleLifecycle( try { const ancestorLifecycles = []; const client = context.core.elasticsearch.dataClient; - - const lifecycleQuery = new LifecycleQuery(legacyEndpointID); + const indexPattern = await indexRetriever.get( + context.core.savedObjects.client, + EndpointAppConstants.EVENT_DATASET + ); + const lifecycleQuery = new LifecycleQuery(indexPattern, legacyEndpointID); const { results: processLifecycle } = await lifecycleQuery.search(client, id); let nextParentID = getParentEntityID(processLifecycle); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts index be83efc39ca4c..b049439207e50 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts @@ -11,6 +11,7 @@ import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public export abstract class ResolverQuery { constructor( + private readonly indexPattern: string, private readonly endpointID?: string, private readonly pagination?: PaginationParams ) {} @@ -26,7 +27,7 @@ export abstract class ResolverQuery { if (this.endpointID) { return this.legacyQuery(this.endpointID, ids, EndpointAppConstants.LEGACY_EVENT_INDEX_NAME); } - return this.query(ids, EndpointAppConstants.EVENT_INDEX_NAME); + return this.query(ids, this.indexPattern); } async search(client: IScopedClusterClient, ...ids: string[]) { diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts index 08a906e2884d6..e73053d53dee0 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/children.test.ts @@ -6,11 +6,17 @@ import { ChildrenQuery } from './children'; import { EndpointAppConstants } from '../../../../common/types'; +export const fakeEventIndexPattern = 'events-endpoint-*'; + describe('children events query', () => { it('generates the correct legacy queries', () => { const timestamp = new Date().getTime(); expect( - new ChildrenQuery('awesome-id', { size: 1, timestamp, eventID: 'foo' }).build('5') + new ChildrenQuery(EndpointAppConstants.LEGACY_EVENT_INDEX_NAME, 'awesome-id', { + size: 1, + timestamp, + eventID: 'foo', + }).build('5') ).toStrictEqual({ body: { query: { @@ -50,7 +56,11 @@ describe('children events query', () => { const timestamp = new Date().getTime(); expect( - new ChildrenQuery(undefined, { size: 1, timestamp, eventID: 'bar' }).build('baz') + new ChildrenQuery(fakeEventIndexPattern, undefined, { + size: 1, + timestamp, + eventID: 'bar', + }).build('baz') ).toStrictEqual({ body: { query: { @@ -88,7 +98,7 @@ describe('children events query', () => { size: 1, sort: [{ '@timestamp': 'asc' }, { 'event.id': 'asc' }], }, - index: EndpointAppConstants.EVENT_INDEX_NAME, + index: fakeEventIndexPattern, }); }); }); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts index b1b47bfb9de7f..8a3955706b278 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/lifecycle.test.ts @@ -5,10 +5,13 @@ */ import { EndpointAppConstants } from '../../../../common/types'; import { LifecycleQuery } from './lifecycle'; +import { fakeEventIndexPattern } from './children.test'; describe('lifecycle query', () => { it('generates the correct legacy queries', () => { - expect(new LifecycleQuery('awesome-id').build('5')).toStrictEqual({ + expect( + new LifecycleQuery(EndpointAppConstants.LEGACY_EVENT_INDEX_NAME, 'awesome-id').build('5') + ).toStrictEqual({ body: { query: { bool: { @@ -32,7 +35,7 @@ describe('lifecycle query', () => { }); it('generates the correct non-legacy queries', () => { - expect(new LifecycleQuery().build('baz')).toStrictEqual({ + expect(new LifecycleQuery(fakeEventIndexPattern).build('baz')).toStrictEqual({ body: { query: { bool: { @@ -57,7 +60,7 @@ describe('lifecycle query', () => { }, sort: [{ '@timestamp': 'asc' }], }, - index: EndpointAppConstants.EVENT_INDEX_NAME, + index: fakeEventIndexPattern, }); }); }); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts b/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts index a91c87274b8dd..5caef935ce621 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/queries/related_events.test.ts @@ -5,12 +5,17 @@ */ import { RelatedEventsQuery } from './related_events'; import { EndpointAppConstants } from '../../../../common/types'; +import { fakeEventIndexPattern } from './children.test'; describe('related events query', () => { it('generates the correct legacy queries', () => { const timestamp = new Date().getTime(); expect( - new RelatedEventsQuery('awesome-id', { size: 1, timestamp, eventID: 'foo' }).build('5') + new RelatedEventsQuery(EndpointAppConstants.LEGACY_EVENT_INDEX_NAME, 'awesome-id', { + size: 1, + timestamp, + eventID: 'foo', + }).build('5') ).toStrictEqual({ body: { query: { @@ -51,7 +56,11 @@ describe('related events query', () => { const timestamp = new Date().getTime(); expect( - new RelatedEventsQuery(undefined, { size: 1, timestamp, eventID: 'bar' }).build('baz') + new RelatedEventsQuery(fakeEventIndexPattern, undefined, { + size: 1, + timestamp, + eventID: 'bar', + }).build('baz') ).toStrictEqual({ body: { query: { @@ -90,7 +99,7 @@ describe('related events query', () => { size: 1, sort: [{ '@timestamp': 'asc' }, { 'event.id': 'asc' }], }, - index: EndpointAppConstants.EVENT_INDEX_NAME, + index: fakeEventIndexPattern, }); }); }); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts b/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts index 804400522065c..d6e9dd586e86c 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts @@ -6,8 +6,10 @@ import { schema } from '@kbn/config-schema'; import { RequestHandler, Logger } from 'kibana/server'; +import { EndpointAppConstants } from '../../../common/types'; import { getPaginationParams } from './utils/pagination'; import { RelatedEventsQuery } from './queries/related_events'; +import { IndexPatternRetriever } from '../../index_pattern'; interface RelatedEventsQueryParams { after?: string; @@ -42,7 +44,8 @@ export const validateRelatedEvents = { }; export function handleRelatedEvents( - log: Logger + log: Logger, + indexRetriever: IndexPatternRetriever ): RequestHandler { return async (context, req, res) => { const { @@ -53,8 +56,12 @@ export function handleRelatedEvents( const pagination = getPaginationParams(limit, after); const client = context.core.elasticsearch.dataClient; + const indexPattern = await indexRetriever.get( + context.core.savedObjects.client, + EndpointAppConstants.EVENT_DATASET + ); // Retrieve the related non-process events for a given process - const relatedEventsQuery = new RelatedEventsQuery(legacyEndpointID, pagination); + const relatedEventsQuery = new RelatedEventsQuery(indexPattern, legacyEndpointID, pagination); const relatedEvents = await relatedEventsQuery.search(client, id); const { total, results: events, nextCursor } = relatedEvents; diff --git a/x-pack/plugins/endpoint/server/test_data/all_metadata_data.json b/x-pack/plugins/endpoint/server/test_data/all_metadata_data.json index 3c8486aa127ea..48952afb33f68 100644 --- a/x-pack/plugins/endpoint/server/test_data/all_metadata_data.json +++ b/x-pack/plugins/endpoint/server/test_data/all_metadata_data.json @@ -1,115 +1,109 @@ { - "took" : 343, - "timed_out" : false, - "_shards" : { - "total" : 1, - "successful" : 1, - "skipped" : 0, - "failed" : 0 + "took": 343, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 }, - "hits" : { - "total" : { - "value" : 4, - "relation" : "eq" + "hits": { + "total": { + "value": 4, + "relation": "eq" }, - "max_score" : null, - "hits" : [ + "max_score": null, + "hits": [ { - "_index" : "endpoint-agent", - "_id" : "WqVo1G8BYQH1gtPUgYkC", - "_score" : null, - "_source" : { - "@timestamp" : 1579816615336, - "event" : { - "created" : "2020-01-23T21:56:55.336Z" + "_index": "metadata-endpoint-default-1", + "_id": "WqVo1G8BYQH1gtPUgYkC", + "_score": null, + "_source": { + "@timestamp": 1579816615336, + "event": { + "created": "2020-01-23T21:56:55.336Z" }, "elastic": { "agent": { "id": "56a75650-3c8a-4e4f-ac17-6dd729c650e2" } }, - "endpoint" : { - "policy" : { - "id" : "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" + "endpoint": { + "policy": { + "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" } }, - "agent" : { - "version" : "6.8.3", - "id" : "56a75650-3c8a-4e4f-ac17-6dd729c650e2", + "agent": { + "version": "6.8.3", + "id": "56a75650-3c8a-4e4f-ac17-6dd729c650e2", "name": "Elastic Endpoint" }, - "host" : { - "id" : "7141a48b-e19f-4ae3-89a0-6e7179a84265", - "hostname" : "larimer-0.example.com", - "ip" : "10.21.48.136", - "mac" : "77-be-30-f0-e8-d6", - "architecture" : "x86_64", - "os" : { - "name" : "windows 6.2", - "full" : "Windows Server 2012", - "version" : "6.2", - "variant" : "Windows Server" + "host": { + "id": "7141a48b-e19f-4ae3-89a0-6e7179a84265", + "hostname": "larimer-0.example.com", + "ip": "10.21.48.136", + "mac": "77-be-30-f0-e8-d6", + "architecture": "x86_64", + "os": { + "name": "windows 6.2", + "full": "Windows Server 2012", + "version": "6.2", + "variant": "Windows Server" } } }, - "fields" : { - "host.id.keyword" : [ - "7141a48b-e19f-4ae3-89a0-6e7179a84265" - ] + "fields": { + "host.id.keyword": ["7141a48b-e19f-4ae3-89a0-6e7179a84265"] }, - "sort" : [ - 1579816615336 - ], - "inner_hits" : { - "most_recent" : { - "hits" : { - "total" : { - "value" : 2, - "relation" : "eq" + "sort": [1579816615336], + "inner_hits": { + "most_recent": { + "hits": { + "total": { + "value": 2, + "relation": "eq" }, - "max_score" : null, - "hits" : [ + "max_score": null, + "hits": [ { - "_index" : "endpoint-agent", - "_id" : "WqVo1G8BYQH1gtPUgYkC", - "_score" : null, - "_source" : { - "@timestamp" : 1579816615336, - "event" : { - "created" : "2020-01-23T21:56:55.336Z" + "_index": "metadata-endpoint-default-1", + "_id": "WqVo1G8BYQH1gtPUgYkC", + "_score": null, + "_source": { + "@timestamp": 1579816615336, + "event": { + "created": "2020-01-23T21:56:55.336Z" }, "elastic": { "agent": { "id": "56a75650-3c8a-4e4f-ac17-6dd729c650e2" } }, - "endpoint" : { - "policy" : { - "id" : "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" + "endpoint": { + "policy": { + "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" } }, - "agent" : { - "version" : "6.8.3", - "id" : "56a75650-3c8a-4e4f-ac17-6dd729c650e2", + "agent": { + "version": "6.8.3", + "id": "56a75650-3c8a-4e4f-ac17-6dd729c650e2", "name": "Elastic Endpoint" }, - "host" : { - "id" : "7141a48b-e19f-4ae3-89a0-6e7179a84265", - "hostname" : "larimer-0.example.com", - "ip" : "10.21.48.136", - "mac" : "77-be-30-f0-e8-d6", - "architecture" : "x86_64", - "os" : { - "name" : "windows 6.2", - "full" : "Windows Server 2012", - "version" : "6.2", - "variant" : "Windows Server" + "host": { + "id": "7141a48b-e19f-4ae3-89a0-6e7179a84265", + "hostname": "larimer-0.example.com", + "ip": "10.21.48.136", + "mac": "77-be-30-f0-e8-d6", + "architecture": "x86_64", + "os": { + "name": "windows 6.2", + "full": "Windows Server 2012", + "version": "6.2", + "variant": "Windows Server" } } }, - "sort" : [ - 1579816615336 - ] + "sort": [1579816615336] } ] } @@ -117,101 +111,95 @@ } }, { - "_index" : "endpoint-agent", - "_id" : "W6Vo1G8BYQH1gtPUgYkC", - "_score" : null, - "_source" : { - "@timestamp" : 1579816615336, - "event" : { - "created" : "2020-01-23T21:56:55.336Z" + "_index": "metadata-endpoint-default-1", + "_id": "W6Vo1G8BYQH1gtPUgYkC", + "_score": null, + "_source": { + "@timestamp": 1579816615336, + "event": { + "created": "2020-01-23T21:56:55.336Z" }, "elastic": { "agent": { "id": "c2d84d8f-d355-40de-8b54-5d318d4d1312" } }, - "endpoint" : { - "policy" : { - "id" : "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" + "endpoint": { + "policy": { + "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" } }, - "agent" : { - "version" : "6.4.3", - "id" : "c2d84d8f-d355-40de-8b54-5d318d4d1312", + "agent": { + "version": "6.4.3", + "id": "c2d84d8f-d355-40de-8b54-5d318d4d1312", "name": "Elastic Endpoint" }, - "host" : { - "id" : "f35ec6c1-6562-45b1-818f-2f14c0854adf", - "hostname" : "hildebrandt-6.example.com", - "ip" : "10.53.92.84", - "mac" : "af-f1-8f-51-25-2a", - "architecture" : "x86_64", - "os" : { - "name" : "windows 10.0", - "full" : "Windows 10", - "version" : "10.0", - "variant" : "Windows Pro" + "host": { + "id": "f35ec6c1-6562-45b1-818f-2f14c0854adf", + "hostname": "hildebrandt-6.example.com", + "ip": "10.53.92.84", + "mac": "af-f1-8f-51-25-2a", + "architecture": "x86_64", + "os": { + "name": "windows 10.0", + "full": "Windows 10", + "version": "10.0", + "variant": "Windows Pro" } } }, - "fields" : { - "host.id.keyword" : [ - "f35ec6c1-6562-45b1-818f-2f14c0854adf" - ] + "fields": { + "host.id.keyword": ["f35ec6c1-6562-45b1-818f-2f14c0854adf"] }, - "sort" : [ - 1579816615336 - ], - "inner_hits" : { - "most_recent" : { - "hits" : { - "total" : { - "value" : 2, - "relation" : "eq" + "sort": [1579816615336], + "inner_hits": { + "most_recent": { + "hits": { + "total": { + "value": 2, + "relation": "eq" }, - "max_score" : null, - "hits" : [ + "max_score": null, + "hits": [ { - "_index" : "endpoint-agent", - "_id" : "W6Vo1G8BYQH1gtPUgYkC", - "_score" : null, - "_source" : { - "@timestamp" : 1579816615336, - "event" : { - "created" : "2020-01-23T21:56:55.336Z" + "_index": "metadata-endpoint-default-1", + "_id": "W6Vo1G8BYQH1gtPUgYkC", + "_score": null, + "_source": { + "@timestamp": 1579816615336, + "event": { + "created": "2020-01-23T21:56:55.336Z" }, "elastic": { "agent": { "id": "c2d84d8f-d355-40de-8b54-5d318d4d1312" } }, - "endpoint" : { - "policy" : { - "id" : "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" + "endpoint": { + "policy": { + "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" } }, - "agent" : { - "version" : "6.4.3", - "id" : "c2d84d8f-d355-40de-8b54-5d318d4d1312", + "agent": { + "version": "6.4.3", + "id": "c2d84d8f-d355-40de-8b54-5d318d4d1312", "name": "Elastic Endpoint" }, - "host" : { - "id" : "f35ec6c1-6562-45b1-818f-2f14c0854adf", - "hostname" : "hildebrandt-6.example.com", - "ip" : "10.53.92.84", - "mac" : "af-f1-8f-51-25-2a", - "architecture" : "x86_64", - "os" : { - "name" : "windows 10.0", - "full" : "Windows 10", - "version" : "10.0", - "variant" : "Windows Pro" + "host": { + "id": "f35ec6c1-6562-45b1-818f-2f14c0854adf", + "hostname": "hildebrandt-6.example.com", + "ip": "10.53.92.84", + "mac": "af-f1-8f-51-25-2a", + "architecture": "x86_64", + "os": { + "name": "windows 10.0", + "full": "Windows 10", + "version": "10.0", + "variant": "Windows Pro" } } }, - "sort" : [ - 1579816615336 - ] + "sort": [1579816615336] } ] } @@ -220,9 +208,9 @@ } ] }, - "aggregations" : { - "total" : { - "value" : 2 + "aggregations": { + "total": { + "value": 2 } } } diff --git a/x-pack/plugins/endpoint/server/types.ts b/x-pack/plugins/endpoint/server/types.ts index 6dc128bd3d61e..46a23060339f4 100644 --- a/x-pack/plugins/endpoint/server/types.ts +++ b/x-pack/plugins/endpoint/server/types.ts @@ -5,11 +5,13 @@ */ import { LoggerFactory } from 'kibana/server'; import { EndpointConfigType } from './config'; +import { IndexPatternRetriever } from './index_pattern'; /** * The context for Endpoint apps. */ export interface EndpointAppContext { + indexPatternRetriever: IndexPatternRetriever; logFactory: LoggerFactory; config(): Promise; } diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 5524e7505d74b..ddb5aa84a9cf8 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -216,8 +216,13 @@ export type PackageInfo = Installable< PackageAdditions >; +export interface InstalledReferences extends SavedObjectAttributes { + references: AssetReference[]; + patterns: Record; +} + export interface Installation extends SavedObjectAttributes { - installed: AssetReference[]; + installed: InstalledReferences; name: string; version: string; } diff --git a/x-pack/plugins/ingest_manager/server/plugin.ts b/x-pack/plugins/ingest_manager/server/plugin.ts index 45c847fe1f68a..b37df701c4127 100644 --- a/x-pack/plugins/ingest_manager/server/plugin.ts +++ b/x-pack/plugins/ingest_manager/server/plugin.ts @@ -11,7 +11,9 @@ import { Plugin, PluginInitializerContext, SavedObjectsServiceStart, -} from 'src/core/server'; + RecursiveReadonly, +} from 'kibana/server'; +import { deepFreeze } from '../../../../src/core/utils'; import { LicensingPluginSetup } from '../../licensing/server'; import { EncryptedSavedObjectsPluginStart } from '../../encrypted_saved_objects/server'; import { SecurityPluginSetup } from '../../security/server'; @@ -38,7 +40,14 @@ import { } from './routes'; import { IngestManagerConfigType } from '../common'; -import { appContextService } from './services'; +import { appContextService, IndexPatternService, IndexPatternSavedObjectService } from './services'; + +/** + * Describes public IngestManager plugin contract returned at the `setup` stage. + */ +export interface IngestManagerSetupContract { + indexPatternService: IndexPatternService; +} export interface IngestManagerSetupDeps { licensing: LicensingPluginSetup; @@ -63,7 +72,7 @@ const allSavedObjectTypes = [ ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, ]; -export class IngestManagerPlugin implements Plugin { +export class IngestManagerPlugin implements Plugin { private config$: Observable; private security: SecurityPluginSetup | undefined; @@ -71,7 +80,10 @@ export class IngestManagerPlugin implements Plugin { this.config$ = this.initializerContext.config.create(); } - public async setup(core: CoreSetup, deps: IngestManagerSetupDeps) { + public async setup( + core: CoreSetup, + deps: IngestManagerSetupDeps + ): Promise> { if (deps.security) { this.security = deps.security; } @@ -130,6 +142,9 @@ export class IngestManagerPlugin implements Plugin { basePath: core.http.basePath, }); } + return deepFreeze({ + indexPatternService: new IndexPatternSavedObjectService(), + }); } public async start( diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index 48f37a4d65ac6..8714b0f3dbdeb 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -129,7 +129,7 @@ export const installPackageHandler: RequestHandler { + if (!datasets) { + return {}; + } + + const patterns: Record = {}; + for (const dataset of datasets) { + patterns[dataset.path] = generateTemplateName(dataset) + '-*'; + } + return patterns; +} + function getBaseTemplate(type: string, templateName: string, mappings: Mappings): IndexTemplate { return { // We need to decide which order we use for the templates diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 82523e37509d1..a63b3854aaa81 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -9,6 +9,7 @@ import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { AssetReference, Installation, + InstalledReferences, KibanaAssetType, CallESAsCurrentUser, DefaultPackages, @@ -18,6 +19,7 @@ import * as Registry from '../registry'; import { getObject } from './get_objects'; import { getInstallation, findInstalledPackageByName } from './index'; import { installTemplates } from '../elasticsearch/template/install'; +import { generateIndexPatterns } from '../elasticsearch/template/template'; import { installPipelines } from '../elasticsearch/ingest_pipeline/install'; import { installILMPolicy } from '../elasticsearch/ilm/install'; @@ -25,7 +27,7 @@ export async function installLatestPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; callCluster: CallESAsCurrentUser; -}): Promise { +}): Promise { const { savedObjectsClient, pkgName, callCluster } = options; try { const latestPackage = await Registry.fetchFindLatestPackage(pkgName); @@ -84,7 +86,7 @@ export async function installPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; callCluster: CallESAsCurrentUser; -}): Promise { +}): Promise { const { savedObjectsClient, pkgkey, callCluster } = options; const registryPackageInfo = await Registry.fetchInfo(pkgkey); const { name: pkgName, version: pkgVersion, internal = false } = registryPackageInfo; @@ -109,17 +111,18 @@ export async function installPackage(options: { installTemplatePromises, ]); - const toSave = res.flat(); + const toSaveRefs: AssetReference[] = res.flat(); + const toSavePatterns = generateIndexPatterns(registryPackageInfo.datasets); // Save those references in the package manager's state saved object - await saveInstallationReferences({ + return await saveInstallationReferences({ savedObjectsClient, pkgkey, pkgName, pkgVersion, internal, - toSave, + toSaveRefs, + toSavePatterns, }); - return toSave; } // TODO: make it an exhaustive list @@ -147,25 +150,42 @@ export async function saveInstallationReferences(options: { pkgName: string; pkgVersion: string; internal: boolean; - toSave: AssetReference[]; + toSaveRefs: AssetReference[]; + toSavePatterns: Record; }) { - const { savedObjectsClient, pkgkey, pkgName, pkgVersion, internal, toSave } = options; + const { + savedObjectsClient, + pkgkey, + pkgName, + pkgVersion, + internal, + toSaveRefs, + toSavePatterns, + } = options; const installation = await getInstallation({ savedObjectsClient, pkgkey }); - const savedRefs = installation?.installed || []; + const savedRefs = installation?.installed.references || []; + const toInstallStreams = Object.assign(installation?.datasetIndexPattern || {}, toSavePatterns); + const mergeRefsReducer = (current: AssetReference[], pending: AssetReference) => { const hasRef = current.find(c => c.id === pending.id && c.type === pending.type); if (!hasRef) current.push(pending); return current; }; - const toInstall = toSave.reduce(mergeRefsReducer, savedRefs); + const toInstallRefs = toSaveRefs.reduce(mergeRefsReducer, savedRefs); + const installed = { references: toInstallRefs, patterns: toInstallStreams }; await savedObjectsClient.create( PACKAGES_SAVED_OBJECT_TYPE, - { installed: toInstall, name: pkgName, version: pkgVersion, internal }, + { + installed, + name: pkgName, + version: pkgVersion, + internal, + }, { id: pkgkey, overwrite: true } ); - return toInstall; + return installed; } async function installKibanaSavedObjects({ diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts index 2e73160453c2b..dd0895ac9a729 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts @@ -18,7 +18,7 @@ export async function removeInstallation(options: { }): Promise { const { savedObjectsClient, pkgkey, callCluster } = options; const installation = await getInstallation({ savedObjectsClient, pkgkey }); - const installedObjects = installation?.installed || []; + const installedObjects = installation?.installed || { references: [], patterns: {} }; // Delete the manager saved object with references to the asset objects // could also update with [] or some other state @@ -28,7 +28,7 @@ export async function removeInstallation(options: { await installIndexPatterns(savedObjectsClient); // Delete the installed assets - const deletePromises = installedObjects.map(async ({ id, type }) => { + const deletePromises = installedObjects.references.map(async ({ id, type }) => { const assetType = type as AssetType; if (savedObjectTypes.includes(assetType)) { savedObjectsClient.delete(assetType, id); @@ -41,7 +41,7 @@ export async function removeInstallation(options: { await Promise.all([...deletePromises]); // successful delete's in SO client return {}. return something more useful - return installedObjects; + return installedObjects.references; } async function deletePipeline(callCluster: CallESAsCurrentUser, id: string): Promise { diff --git a/x-pack/plugins/ingest_manager/server/services/index.ts b/x-pack/plugins/ingest_manager/server/services/index.ts index dd0c898afa425..0b5ef90572524 100644 --- a/x-pack/plugins/ingest_manager/server/services/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ export { appContextService } from './app_context'; +export { IndexPatternService, IndexPatternSavedObjectService } from './index_pattern'; // Saved object services export { datasourceService } from './datasource'; diff --git a/x-pack/plugins/ingest_manager/server/services/index_pattern.ts b/x-pack/plugins/ingest_manager/server/services/index_pattern.ts new file mode 100644 index 0000000000000..c1a881eaf36cb --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/index_pattern.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { SavedObjectsClientContract } from 'kibana/server'; +import * as Registry from './epm/registry'; +import { getInstallationObject, findInstalledPackageByName } from './epm/packages/get'; +import { Installation } from '../types'; + +export interface IndexPatternService { + get( + savedObjectsClient: SavedObjectsClientContract, + pkgName: string, + datasetPath: string, + version?: string + ): Promise; +} + +export class IndexPatternSavedObjectService implements IndexPatternService { + public async get( + savedObjectsClient: SavedObjectsClientContract, + pkgName: string, + datasetPath: string, + version?: string + ): Promise { + let installation: Installation | undefined; + if (version) { + const pkgkey = Registry.pkgToPkgKey({ + name: pkgName, + version, + }); + installation = (await getInstallationObject({ savedObjectsClient, pkgkey }))?.attributes; + } else { + installation = await findInstalledPackageByName({ savedObjectsClient, pkgName }); + } + return installation?.installed.patterns[datasetPath]; + } +} diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 1cd5622c0c7b0..aed94f75333ae 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -28,6 +28,7 @@ export { EnrollmentAPIKey, EnrollmentAPIKeySOAttributes, Installation, + InstalledReferences, InstallationStatus, PackageInfo, RegistryVarsEntry, diff --git a/x-pack/test/api_integration/apis/endpoint/index.ts b/x-pack/test/api_integration/apis/endpoint/index.ts index 4ffd0c3b6044b..1bba84bd2ed6a 100644 --- a/x-pack/test/api_integration/apis/endpoint/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/index.ts @@ -9,6 +9,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function endpointAPIIntegrationTests({ loadTestFile }: FtrProviderContext) { describe('Endpoint plugin', function() { this.tags(['endpoint']); + loadTestFile(require.resolve('./index_pattern')); loadTestFile(require.resolve('./resolver')); loadTestFile(require.resolve('./metadata')); loadTestFile(require.resolve('./alerts')); diff --git a/x-pack/test/api_integration/apis/endpoint/index_pattern.ts b/x-pack/test/api_integration/apis/endpoint/index_pattern.ts new file mode 100644 index 0000000000000..530f542a41d99 --- /dev/null +++ b/x-pack/test/api_integration/apis/endpoint/index_pattern.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect/expect.js'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + + describe('Endpoint index pattern API', () => { + it('should retrieve the index pattern for events', async () => { + const { body } = await supertest + .get('/api/endpoint/index_pattern/events') + .set('kbn-xsrf', 'xxx') + .expect(200); + expect(body.indexPattern).to.eql('events-endpoint-*'); + }); + + it('should retrieve the index pattern for metadata', async () => { + const { body } = await supertest + .get('/api/endpoint/index_pattern/metadata') + .set('kbn-xsrf', 'xxx') + .expect(200); + expect(body.indexPattern).to.eql('metrics-endpoint-*'); + }); + + it('should not retrieve the index pattern for an invalid key', async () => { + await supertest + .get('/api/endpoint/index_pattern/blah') + .set('kbn-xsrf', 'xxx') + .expect(404); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/endpoint/metadata.ts b/x-pack/test/api_integration/apis/endpoint/metadata.ts index ad495d4505404..deb9b882b202a 100644 --- a/x-pack/test/api_integration/apis/endpoint/metadata.ts +++ b/x-pack/test/api_integration/apis/endpoint/metadata.ts @@ -159,7 +159,7 @@ export default function({ getService }: FtrProviderContext) { .post('/api/endpoint/metadata') .set('kbn-xsrf', 'xxx') .send({ - filter: `host.os.variant.keyword:${variantValue}`, + filter: `host.os.variant:${variantValue}`, }) .expect(200); expect(body.total).to.eql(2); diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index c1e9240c09951..e8d336e875b99 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -17,8 +17,8 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC const esArchiver = getService('esArchiver'); describe('Resolver', () => { - before(() => esArchiver.load('endpoint/resolver/api_feature')); - after(() => esArchiver.unload('endpoint/resolver/api_feature')); + before(async () => await esArchiver.load('endpoint/resolver/api_feature')); + after(async () => await esArchiver.unload('endpoint/resolver/api_feature')); describe('related events endpoint', () => { const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; From 8cbfa4230471f7070a5fa63adf9a7575ec3e0242 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 13:02:53 -0400 Subject: [PATCH 06/27] Missed a file --- x-pack/plugins/ingest_manager/server/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/ingest_manager/server/index.ts b/x-pack/plugins/ingest_manager/server/index.ts index df7c3d7cf0fbf..8bc6ae788f070 100644 --- a/x-pack/plugins/ingest_manager/server/index.ts +++ b/x-pack/plugins/ingest_manager/server/index.ts @@ -7,6 +7,9 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { PluginInitializerContext } from 'src/core/server'; import { IngestManagerPlugin } from './plugin'; +export { IndexPatternService } from './services'; +export { IngestManagerSetupContract } from './plugin'; + export const config = { exposeToBrowser: { epm: true, From a1f2bd0fd492c6e034a56997daa83107a1be49b9 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 13:11:43 -0400 Subject: [PATCH 07/27] Pointing resolver tests at endpoint functional tests --- x-pack/test/plugin_functional/config.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test/plugin_functional/config.ts b/x-pack/test/plugin_functional/config.ts index 6c3c496da71f6..aa3c9bd24842a 100644 --- a/x-pack/test/plugin_functional/config.ts +++ b/x-pack/test/plugin_functional/config.ts @@ -14,7 +14,9 @@ import { pageObjects } from './page_objects'; /* eslint-disable import/no-default-export */ export default async function({ readConfigFile }: FtrConfigProviderContext) { - const xpackFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js')); + const xpackFunctionalConfig = await readConfigFile( + require.resolve('../functional_endpoint/config.ts') + ); // Find all folders in ./plugins since we treat all them as plugin folder const allFiles = fs.readdirSync(resolve(__dirname, 'plugins')); From f13231a1d5386e55c0ce07a31a3c7c7dc11d6818 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 13:21:51 -0400 Subject: [PATCH 08/27] Pointing space tests at the endpoint functional directory --- .../apps/endpoint/feature_controls/endpoint_spaces.ts | 2 +- .../functional_endpoint/apps/endpoint/feature_controls/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/endpoint_spaces.ts b/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/endpoint_spaces.ts index ba2e7a248a9dd..c543046031e9f 100644 --- a/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/endpoint_spaces.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/endpoint_spaces.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../../ftr_provider_context'; export default function({ getPageObjects, getService }: FtrProviderContext) { const pageObjects = getPageObjects(['common']); diff --git a/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/index.ts b/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/index.ts index 46fde1596b0cf..5f7e611fd966c 100644 --- a/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/index.ts +++ b/x-pack/test/functional_endpoint/apps/endpoint/feature_controls/index.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { FtrProviderContext } from '../../../../functional/ftr_provider_context'; +import { FtrProviderContext } from '../../../ftr_provider_context'; export default function({ loadTestFile }: FtrProviderContext) { describe('feature controls', function() { From 0140395b6cbc85e51e79b80fb81845266de307dc Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 14:27:10 -0400 Subject: [PATCH 09/27] Adding ingest service to do setup and tests for 500s --- x-pack/scripts/functional_tests.js | 1 + .../api_integration/apis/endpoint/index.ts | 9 +- x-pack/test/api_integration/services/index.ts | 2 + .../services/ingest_manager.ts | 28 +++ .../apis/alerts.ts | 30 ++++ .../apis/index.ts | 16 ++ .../apis/index_pattern.ts | 19 +++ .../apis/metadata.ts | 22 +++ .../config.ts | 19 +++ .../ftr_provider_context.d.ts | 11 ++ .../endpoint/metadata/api_feature/data.json | 18 +- .../metadata/api_feature/mappings.json | 159 ++++++++---------- 12 files changed, 236 insertions(+), 98 deletions(-) create mode 100644 x-pack/test/api_integration/services/ingest_manager.ts create mode 100644 x-pack/test/endpoint_api_integration_no_ingest/apis/alerts.ts create mode 100644 x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts create mode 100644 x-pack/test/endpoint_api_integration_no_ingest/apis/index_pattern.ts create mode 100644 x-pack/test/endpoint_api_integration_no_ingest/apis/metadata.ts create mode 100644 x-pack/test/endpoint_api_integration_no_ingest/config.ts create mode 100644 x-pack/test/endpoint_api_integration_no_ingest/ftr_provider_context.d.ts diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index 061c9e4a0d921..f9926af10cbfa 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -45,6 +45,7 @@ const onlyNotInCoverageTests = [ require.resolve('../test/licensing_plugin/config.legacy.ts'), require.resolve('../test/functional_endpoint_ingest_failure/config.ts'), require.resolve('../test/functional_endpoint/config.ts'), + require.resolve('../test/endpoint_api_integration_no_ingest/config.ts'), ]; require('@kbn/plugin-helpers').babelRegister(); diff --git a/x-pack/test/api_integration/apis/endpoint/index.ts b/x-pack/test/api_integration/apis/endpoint/index.ts index 1bba84bd2ed6a..0a5f9aa595b8a 100644 --- a/x-pack/test/api_integration/apis/endpoint/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/index.ts @@ -6,9 +6,16 @@ import { FtrProviderContext } from '../../ftr_provider_context'; -export default function endpointAPIIntegrationTests({ loadTestFile }: FtrProviderContext) { +export default function endpointAPIIntegrationTests({ + loadTestFile, + getService, +}: FtrProviderContext) { describe('Endpoint plugin', function() { + const ingestManager = getService('ingestManager'); this.tags(['endpoint']); + before(async () => { + await ingestManager.setup(); + }); loadTestFile(require.resolve('./index_pattern')); loadTestFile(require.resolve('./resolver')); loadTestFile(require.resolve('./metadata')); diff --git a/x-pack/test/api_integration/services/index.ts b/x-pack/test/api_integration/services/index.ts index 9c945f557a2d8..84b8476bd1dd1 100644 --- a/x-pack/test/api_integration/services/index.ts +++ b/x-pack/test/api_integration/services/index.ts @@ -22,6 +22,7 @@ import { import { SiemGraphQLClientProvider, SiemGraphQLClientFactoryProvider } from './siem_graphql_client'; import { InfraOpsSourceConfigurationProvider } from './infraops_source_configuration'; import { MachineLearningProvider } from './ml'; +import { IngestManagerProvider } from './ingest_manager'; export const services = { ...commonServices, @@ -39,4 +40,5 @@ export const services = { supertestWithoutAuth: SupertestWithoutAuthProvider, usageAPI: UsageAPIProvider, ml: MachineLearningProvider, + ingestManager: IngestManagerProvider, }; diff --git a/x-pack/test/api_integration/services/ingest_manager.ts b/x-pack/test/api_integration/services/ingest_manager.ts new file mode 100644 index 0000000000000..2b70a20ca0362 --- /dev/null +++ b/x-pack/test/api_integration/services/ingest_manager.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { FtrProviderContext } from '../ftr_provider_context'; +import { setupRouteService, fleetSetupRouteService } from '../../../plugins/ingest_manager/common'; + +export function IngestManagerProvider({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + return { + async setup() { + const headers = { accept: 'application/json', 'kbn-xsrf': 'some-xsrf-token' }; + + const { body } = await supertest + .get(fleetSetupRouteService.getFleetSetupPath()) + .set(headers) + .expect(200); + + if (!body.isInitialized) { + await supertest + .post(setupRouteService.getSetupPath()) + .set(headers) + .expect(200); + } + }, + }; +} diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/alerts.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/alerts.ts new file mode 100644 index 0000000000000..eb50f3cc0ab06 --- /dev/null +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/alerts.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + + describe('Endpoint alert API without ingest manager initialized', () => { + before(async () => { + await esArchiver.load('endpoint/alerts/api_feature'); + await esArchiver.load('endpoint/alerts/host_api_feature'); + }); + + after(async () => { + await esArchiver.unload('endpoint/alerts/api_feature'); + await esArchiver.unload('endpoint/alerts/host_api_feature'); + }); + + it('should return zero results', async () => { + await supertest + .get('/api/endpoint/alerts') + .set('kbn-xsrf', 'xxx') + .expect(500); + }); + }); +} diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts new file mode 100644 index 0000000000000..985ac4852c77b --- /dev/null +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function endpointAPIIntegrationTests({ loadTestFile }: FtrProviderContext) { + describe('Endpoint plugin', function() { + this.tags(['endpoint']); + loadTestFile(require.resolve('./index_pattern')); + loadTestFile(require.resolve('./metadata')); + loadTestFile(require.resolve('./alerts')); + }); +} diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/index_pattern.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/index_pattern.ts new file mode 100644 index 0000000000000..b0934e4da7da0 --- /dev/null +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/index_pattern.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + + describe('Endpoint index pattern API without ingest manager initialized', () => { + it('should retrieve the index pattern for events', async () => { + await supertest + .get('/api/endpoint/index_pattern/events') + .set('kbn-xsrf', 'xxx') + .expect(404); + }); + }); +} diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/metadata.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/metadata.ts new file mode 100644 index 0000000000000..15e9de5da8234 --- /dev/null +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/metadata.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + describe('test metadata api when ingest manager is not initialized', () => { + before(async () => await esArchiver.load('endpoint/metadata/api_feature')); + after(async () => await esArchiver.unload('endpoint/metadata/api_feature')); + it('metadata api should return no results', async () => { + await supertest + .post('/api/endpoint/metadata') + .set('kbn-xsrf', 'xxx') + .send() + .expect(500); + }); + }); +} diff --git a/x-pack/test/endpoint_api_integration_no_ingest/config.ts b/x-pack/test/endpoint_api_integration_no_ingest/config.ts new file mode 100644 index 0000000000000..bf8b68a7e991c --- /dev/null +++ b/x-pack/test/endpoint_api_integration_no_ingest/config.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; + +export default async function({ readConfigFile }: FtrConfigProviderContext) { + const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.js')); + + return { + ...xPackAPITestsConfig.getAll(), + testFiles: [require.resolve('./apis')], + junit: { + reportName: 'X-Pack Endpoint API Integration Without Ingest Tests', + }, + }; +} diff --git a/x-pack/test/endpoint_api_integration_no_ingest/ftr_provider_context.d.ts b/x-pack/test/endpoint_api_integration_no_ingest/ftr_provider_context.d.ts new file mode 100644 index 0000000000000..e3add3748f56d --- /dev/null +++ b/x-pack/test/endpoint_api_integration_no_ingest/ftr_provider_context.d.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GenericFtrProviderContext } from '@kbn/test/types/ftr'; + +import { services } from './services'; + +export type FtrProviderContext = GenericFtrProviderContext; diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json index 2046f46db9f53..d3617dc236375 100644 --- a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json @@ -2,7 +2,7 @@ "type": "doc", "value": { "id": "3KVN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579881969541, "agent": { @@ -51,7 +51,7 @@ "type": "doc", "value": { "id": "3aVN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579881969541, "agent": { @@ -99,7 +99,7 @@ "type": "doc", "value": { "id": "3qVN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579881969541, "agent": { @@ -145,7 +145,7 @@ "type": "doc", "value": { "id": "36VN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579878369541, "agent": { @@ -194,7 +194,7 @@ "type": "doc", "value": { "id": "4KVN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579878369541, "agent": { @@ -241,7 +241,7 @@ "type": "doc", "value": { "id": "4aVN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579878369541, "agent": { @@ -288,7 +288,7 @@ "type": "doc", "value": { "id": "4qVN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579874769541, "agent": { @@ -336,7 +336,7 @@ "type": "doc", "value": { "id": "46VN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579874769541, "agent": { @@ -383,7 +383,7 @@ "type": "doc", "value": { "id": "5KVN2G8BYQH1gtPUuYk7", - "index": "endpoint-agent-1", + "index": "metrics-endpoint-default-1", "source": { "@timestamp": 1579874769541, "agent": { diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/mappings.json b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/mappings.json index c9a6c183f0489..743764c1b0553 100644 --- a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/mappings.json @@ -1,13 +1,27 @@ { "type": "index", "value": { - "aliases": { - }, - "index": "endpoint-agent-1", + "aliases": {}, + "index": "metrics-endpoint-default-1", "mappings": { + "_meta": { + "version": "1.5.0-dev" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], "properties": { "@timestamp": { - "type": "long" + "type": "date" }, "elastic": { "properties": { @@ -29,22 +43,24 @@ "agent": { "properties": { "id": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" }, "version": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -53,109 +69,71 @@ "policy": { "properties": { "id": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - }, - "name": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" } - } + }, + "type": "object" } } }, "event": { "properties": { "created": { - "type": "long" + "type": "date" } } }, "host": { "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, "hostname": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "id": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "ip": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "type": "ip" }, "mac": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "os": { "properties": { "full": { "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" + "text": { + "norms": false, + "type": "text" } }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "name": { "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" + "text": { + "norms": false, + "type": "text" } }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "variant": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "version": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" } } } @@ -163,10 +141,15 @@ } } }, + "order": 1, "settings": { "index": { - "number_of_replicas": "1", - "number_of_shards": "1" + "mapping": { + "total_fields": { + "limit": 10000 + } + }, + "refresh_interval": "5s" } } } From 6c7b7b42904b965b32d83004fe1ac6c299df5f2d Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 15:33:59 -0400 Subject: [PATCH 10/27] Correcting services path --- .../ftr_provider_context.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/endpoint_api_integration_no_ingest/ftr_provider_context.d.ts b/x-pack/test/endpoint_api_integration_no_ingest/ftr_provider_context.d.ts index e3add3748f56d..2751dbcdc6539 100644 --- a/x-pack/test/endpoint_api_integration_no_ingest/ftr_provider_context.d.ts +++ b/x-pack/test/endpoint_api_integration_no_ingest/ftr_provider_context.d.ts @@ -6,6 +6,6 @@ import { GenericFtrProviderContext } from '@kbn/test/types/ftr'; -import { services } from './services'; +import { services } from '../api_integration/services'; export type FtrProviderContext = GenericFtrProviderContext; From 661f860f6634ff66aea38dbba3f4d83fb72eb4af Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 15:38:29 -0400 Subject: [PATCH 11/27] Adding jest test names --- x-pack/test/functional_endpoint/config.ts | 3 +++ x-pack/test/functional_endpoint_ingest_failure/config.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/x-pack/test/functional_endpoint/config.ts b/x-pack/test/functional_endpoint/config.ts index b44df594b65bf..37bf57b67b47e 100644 --- a/x-pack/test/functional_endpoint/config.ts +++ b/x-pack/test/functional_endpoint/config.ts @@ -15,6 +15,9 @@ export default async function({ readConfigFile }: FtrConfigProviderContext) { ...xpackFunctionalConfig.getAll(), pageObjects, testFiles: [resolve(__dirname, './apps/endpoint')], + junit: { + reportName: 'X-Pack Endpoint Functional Tests', + }, apps: { ...xpackFunctionalConfig.get('apps'), endpoint: { diff --git a/x-pack/test/functional_endpoint_ingest_failure/config.ts b/x-pack/test/functional_endpoint_ingest_failure/config.ts index 347cd0192bbdb..a2055a4bbdc18 100644 --- a/x-pack/test/functional_endpoint_ingest_failure/config.ts +++ b/x-pack/test/functional_endpoint_ingest_failure/config.ts @@ -15,6 +15,9 @@ export default async function({ readConfigFile }: FtrConfigProviderContext) { return { ...xpackFunctionalConfig.getAll(), testFiles: [resolve(__dirname, './apps/endpoint')], + junit: { + reportName: 'X-Pack Endpoint Without Ingest Functional Tests', + }, kbnTestServer: { ...xpackFunctionalConfig.get('kbnTestServer'), serverArgs: [ From cf1a071d7294f2b66eb9f9f5c3a51c262632d77f Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 8 Apr 2020 16:47:02 -0400 Subject: [PATCH 12/27] Updating es archives with the correct mapping and index names --- .../alerts/host_api_feature/data.json.gz | Bin 732 -> 753 bytes .../alerts/host_api_feature/mappings.json | 172 +++++++++--------- .../metadata/api_feature/mappings.json | 12 +- 3 files changed, 88 insertions(+), 96 deletions(-) diff --git a/x-pack/test/functional/es_archives/endpoint/alerts/host_api_feature/data.json.gz b/x-pack/test/functional/es_archives/endpoint/alerts/host_api_feature/data.json.gz index 94a96c54ee9cb66afbd74d89be6a774720c8c94e..a71281c0ecfec93e0a51aa3473dbbbf565a10f9b 100644 GIT binary patch literal 753 zcmVQOZ*BnHl}k_CKoEfM`4v&ljJ0p?_EbZuIH0N( zQc+d)u+PM8Y)6h0post8bs!musT1Ah2*=*TtobuWm=8c1(tp{dF z3Wgy8QdowyMyW(*bo6Mo^z}Ti%GOjQG*P36rMt~hYtMI2n^n`?-t5RWKAezDAw^H& zlt=~a1Z9wAjDb9%Bx9I;&eCklhiF&oVwLdAyqOG7moc`y9FH~{cYUXpz4&eIb>m$( zWj)hLHSbuqlC=kC@g1}&!lbbL&tf}O_rY0a9JVF^PE`ubc?^nJ0))ww8b!I3XveB8 z;?M11F-kCsPl^)4(%nR~YnUbk#Ygp)BuykvFcXid`L%Uap>6jp9(zCmX5baXS{St7 zvYK}*85Yy&eML%FUCoCVzK*RjBnXrJrMr4i6iG*6sq0*?iq+Yyqqg0MWKZn}>;JpK z-C%HgIH$*^!|8ZDdrM9Q7YA_4ov@w-px82CK8S2DNcmw^Gjqx zC{L(7m{LZggJ3KK0z!lCkk1(i;3i&6;2e1NT81%t9^L3XdXe3)b@e2;s}24mxwc2{ z zag-NO#DDKPkdW8`B1A7pu@%qG?9S`kx$Shjz4^@~^geWZzH+^zTCg<3MJ>5aAL+C7 z_3F*}mk~KDj{T=xT%P|v|M4w;U#Zs`V;`<-r7-rBsvOV3^h0{nwZg1gOx@#_N%C=C zj>2qiMw29^LUF=5V}c8Pv{qT|v1PMyC757&upXju0l32)EGH6_betMPaz$+K=+SB! zrn9md*QPRKneII--EEFq`@MVGq#Bm)W=FR1;RFj!G&_Y;R2ujRDWE7g2X(@*;FNzZ z@?y(}WH&aWRhlo$`Jj8cOtF>Yes803*LG^zir-CxnFrrYS<7@#%^H@qWbMIO`VYpA z;$Y;&pT%~n?t^pOdg^Qhn(G|64-_;y1mr4bRx_a#*|DmN_;Wj0N-|2)lama2zMF`8 zjj{|$dUR_^u}sm7a`~8=Ut33wjBB1HQV(dL9D;^a3rqG}RtZlOBZ+qMjZzp^3*)kBjhFol8_Nv=_l!wnR;Jzrj)&g&Nv-nH8-&`Z8`%r5rt+D zgP_1oonWLXOqDJ^W*kLw0G(Q8HJr2sAPP+EW>U7PMU^2F8B+(d$yss`tb>Rk4VVV+ zf`bHM(`5wSL*TC^my+j!P0o`SiTygQp2T&v!5i}GTD%S}{sXUCz07OwqK>&?fXc{p z+0y{lash-0V^Bq^Jb8xKcQ#E^%3i~(yZ$Ts!&I2bo@r`>B7`uITGkgLiK*WWg{Zk> zXgLpByvQq88BKE$5%&z!R0mAI_6oszo-%SUuPW3yS`BDG3Gk}HD*;VXj{v4oTSNUm OM(sbt^Zzdg3jhEPfL<^F diff --git a/x-pack/test/functional/es_archives/endpoint/alerts/host_api_feature/mappings.json b/x-pack/test/functional/es_archives/endpoint/alerts/host_api_feature/mappings.json index 61ddf3c4e65db..f9d5de0d0a94c 100644 --- a/x-pack/test/functional/es_archives/endpoint/alerts/host_api_feature/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/alerts/host_api_feature/mappings.json @@ -1,33 +1,62 @@ { "type": "index", "value": { - "aliases": { - }, - "index": "endpoint-agent-1", + "aliases": {}, + "index": "metrics-endpoint-default-1", "mappings": { + "_meta": { + "version": "1.5.0-dev" + }, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], "properties": { "@timestamp": { - "type": "long" + "type": "date" }, - "agent": { + "elastic": { "properties": { - "id": { - "fields": { - "keyword": { - "ignore_above": 256, + "agent": { + "properties": { + "id": { + "ignore_above": 1024, "type": "keyword" } }, - "type": "text" + "type": "object" + } + } + }, + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" }, "version": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -36,109 +65,71 @@ "policy": { "properties": { "id": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" - }, - "name": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" } - } + }, + "type": "object" } } }, "event": { "properties": { "created": { - "type": "long" + "type": "date" } } }, "host": { "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, "hostname": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "id": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "ip": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "type": "ip" }, "mac": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "os": { "properties": { "full": { "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" + "text": { + "norms": false, + "type": "text" } }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "name": { "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" + "text": { + "norms": false, + "type": "text" } }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "variant": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "version": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" } } } @@ -146,11 +137,16 @@ } } }, + "order": 1, "settings": { "index": { - "number_of_replicas": "1", - "number_of_shards": "1" + "mapping": { + "total_fields": { + "limit": 10000 + } + }, + "refresh_interval": "5s" } } } -} \ No newline at end of file +} diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/mappings.json b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/mappings.json index 743764c1b0553..f9d5de0d0a94c 100644 --- a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/mappings.json @@ -28,15 +28,11 @@ "agent": { "properties": { "id": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "ignore_above": 1024, + "type": "keyword" } - } + }, + "type": "object" } } }, From 9ff1e7bc602409fa0385d5740db27e56c789483c Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Thu, 9 Apr 2020 09:07:35 -0400 Subject: [PATCH 13/27] Fixing import error --- x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts index 32d5c75b50b80..6d1be57422acc 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts @@ -25,7 +25,7 @@ import { SearchResponse } from 'elasticsearch'; import { registerEndpointRoutes } from './index'; import { EndpointConfigSchema } from '../../config'; import { FakeIndexPatternRetriever } from '../../plugin.test'; -import * as data from '../test_data/all_metadata_data.json'; +import * as data from '../../test_data/all_metadata_data.json'; describe('test endpoint route', () => { let routerMock: jest.Mocked; From 3599f4642faaa92f8e6052256cca83e10a71a499 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Thu, 9 Apr 2020 10:46:18 -0400 Subject: [PATCH 14/27] Adding resolver tests to code owners --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e707250ff3261..6d9d577eefdf1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -202,9 +202,12 @@ # Endpoint /x-pack/plugins/endpoint/ @elastic/endpoint-app-team /x-pack/test/api_integration/apis/endpoint/ @elastic/endpoint-app-team +/x-pack/test/endpoint_api_integration_no_ingest/ @elastic/endpoint-app-team /x-pack/test/functional_endpoint/ @elastic/endpoint-app-team /x-pack/test/functional_endpoint_ingest_failure/ @elastic/endpoint-app-team /x-pack/test/functional/es_archives/endpoint/ @elastic/endpoint-app-team +/x-pack/test/plugin_functional/plugins/resolver_test/ @elastic/endpoint-app-team +/x-pack/test/plugin_functional/test_suites/resolver/ @elastic/endpoint-app-team # SIEM /x-pack/legacy/plugins/siem/ @elastic/siem From 6c221463f752ce04c84acf150f1da75ddda57507 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Thu, 9 Apr 2020 11:00:15 -0400 Subject: [PATCH 15/27] enabling epm flag for functional tests --- x-pack/test/functional_endpoint/config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/functional_endpoint/config.ts b/x-pack/test/functional_endpoint/config.ts index 37bf57b67b47e..6ae78ab9d48ac 100644 --- a/x-pack/test/functional_endpoint/config.ts +++ b/x-pack/test/functional_endpoint/config.ts @@ -30,6 +30,7 @@ export default async function({ readConfigFile }: FtrConfigProviderContext) { ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'), '--xpack.endpoint.enabled=true', '--xpack.ingestManager.enabled=true', + '--xpack.ingestManager.epm.enabled=true', '--xpack.ingestManager.fleet.enabled=true', ], }, From 6457c0f0b1f6ec305d4d0abf8433c00cf3859139 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Thu, 9 Apr 2020 11:09:15 -0400 Subject: [PATCH 16/27] adding correct tag to test --- x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts index 985ac4852c77b..6110f398df5a0 100644 --- a/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts @@ -8,7 +8,7 @@ import { FtrProviderContext } from '../ftr_provider_context'; export default function endpointAPIIntegrationTests({ loadTestFile }: FtrProviderContext) { describe('Endpoint plugin', function() { - this.tags(['endpoint']); + this.tags('ciGroup7'); loadTestFile(require.resolve('./index_pattern')); loadTestFile(require.resolve('./metadata')); loadTestFile(require.resolve('./alerts')); From 4d2e8ce8cacc5f2e39d8416a4e6bbe8cdf60dc5e Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Thu, 9 Apr 2020 16:18:32 -0400 Subject: [PATCH 17/27] Removing the version information and unneeded xsrf --- .../endpoint/common/schema/index_pattern.ts | 3 --- x-pack/plugins/endpoint/common/types.ts | 17 +-------------- .../plugins/endpoint/server/index_pattern.ts | 13 ++++-------- x-pack/plugins/endpoint/server/plugin.test.ts | 10 +++------ .../endpoint/server/routes/index_pattern.ts | 14 +++++-------- .../server/services/index_pattern.ts | 21 ++++--------------- .../apis/endpoint/index_pattern.ts | 15 +++---------- .../apis/alerts.ts | 7 ++----- .../apis/index_pattern.ts | 7 ++----- .../apis/metadata.ts | 2 +- 10 files changed, 25 insertions(+), 84 deletions(-) diff --git a/x-pack/plugins/endpoint/common/schema/index_pattern.ts b/x-pack/plugins/endpoint/common/schema/index_pattern.ts index 32144f820eca6..2809004f88c6e 100644 --- a/x-pack/plugins/endpoint/common/schema/index_pattern.ts +++ b/x-pack/plugins/endpoint/common/schema/index_pattern.ts @@ -7,6 +7,3 @@ import { schema } from '@kbn/config-schema'; export const indexPatternGetParamsSchema = schema.object({ datasetPath: schema.string() }); -export const indexPatternGetQueryParamsSchema = schema.object({ - version: schema.maybe(schema.string()), -}); diff --git a/x-pack/plugins/endpoint/common/types.ts b/x-pack/plugins/endpoint/common/types.ts index 9de08cd061734..617ed7613eadd 100644 --- a/x-pack/plugins/endpoint/common/types.ts +++ b/x-pack/plugins/endpoint/common/types.ts @@ -7,10 +7,7 @@ import { SearchResponse } from 'elasticsearch'; import { TypeOf } from '@kbn/config-schema'; import { alertingIndexGetQuerySchema } from './schema/alert_index'; -import { - indexPatternGetParamsSchema, - indexPatternGetQueryParamsSchema, -} from './schema/index_pattern'; +import { indexPatternGetParamsSchema } from './schema/index_pattern'; /** * A deep readonly type that will make all children of a given object readonly recursively @@ -456,15 +453,3 @@ export type AlertingIndexGetQueryResult = TypeOf; - -/** - * Query params to pass to the index pattern API when fetching an index pattern. - */ -export type IndexPatternGetQueryParamsInput = KbnConfigSchemaInputTypeOf< - TypeOf ->; - -/** - * Result of the validated query params when handling an index pattern request. - */ -export type IndexPatternGetQueryParamsResult = TypeOf; diff --git a/x-pack/plugins/endpoint/server/index_pattern.ts b/x-pack/plugins/endpoint/server/index_pattern.ts index 38d69aedf3e81..0dc5b928a96f1 100644 --- a/x-pack/plugins/endpoint/server/index_pattern.ts +++ b/x-pack/plugins/endpoint/server/index_pattern.ts @@ -9,7 +9,7 @@ import { IndexPatternService } from '../../ingest_manager/server'; const endpointPackageName = 'endpoint'; export interface IndexPatternRetriever { - get(client: SavedObjectsClientContract, datasetPath: string, version?: string): Promise; + get(client: SavedObjectsClientContract, datasetPath: string): Promise; } export class IngestIndexPatternRetriever implements IndexPatternRetriever { @@ -19,16 +19,11 @@ export class IngestIndexPatternRetriever implements IndexPatternRetriever { } async get(client: SavedObjectsClientContract, datasetPath: string, version?: string) { - const pattern = await this.service.get(client, endpointPackageName, datasetPath, version); + const pattern = await this.service.get(client, endpointPackageName, datasetPath); if (!pattern) { - const versionStr = version || 'none'; - this.log.warn( - `Failed to retrieve index pattern from ingest manager dataset: ${datasetPath} version: ${versionStr}` - ); - throw new Error( - `Unable to retrieve the index pattern for dataset: ${datasetPath} version: ${version}` - ); + this.log.warn(`Failed to retrieve index pattern from ingest manager dataset: ${datasetPath}`); + throw new Error(`Unable to retrieve the index pattern for dataset: ${datasetPath}`); } return pattern; diff --git a/x-pack/plugins/endpoint/server/plugin.test.ts b/x-pack/plugins/endpoint/server/plugin.test.ts index fd308c11cdbf7..8e889ed0f1601 100644 --- a/x-pack/plugins/endpoint/server/plugin.test.ts +++ b/x-pack/plugins/endpoint/server/plugin.test.ts @@ -10,13 +10,10 @@ import { EndpointPlugin, EndpointPluginSetupDependencies } from './plugin'; import { coreMock } from '../../../../src/core/server/mocks'; import { PluginSetupContract } from '../../features/server'; import { IndexPatternService } from '../../ingest_manager/server'; +import { IndexPatternRetriever } from './index_pattern'; export const MetadataIndexPattern = 'metadata-endpoint-*'; -export interface IndexPatternRetriever { - get(client: SavedObjectsClientContract, datasetPath: string, version?: string): Promise; -} - export class FakeIndexPatternRetriever implements IndexPatternRetriever { constructor(private readonly indexPattern: string) {} @@ -24,7 +21,7 @@ export class FakeIndexPatternRetriever implements IndexPatternRetriever { return new FakeIndexPatternRetriever(MetadataIndexPattern); } - async get(client: SavedObjectsClientContract, datasetPath: string, version?: string) { + async get(client: SavedObjectsClientContract, datasetPath: string) { return this.indexPattern; } } @@ -35,8 +32,7 @@ export class FakeIndexPatternService implements IndexPatternService { async get( savedObjectsClient: SavedObjectsClientContract, pkgName: string, - datasetPath: string, - version?: string + datasetPath: string ): Promise { return this.indexPattern; } diff --git a/x-pack/plugins/endpoint/server/routes/index_pattern.ts b/x-pack/plugins/endpoint/server/routes/index_pattern.ts index 79494ece5e956..69ff27a9d003d 100644 --- a/x-pack/plugins/endpoint/server/routes/index_pattern.ts +++ b/x-pack/plugins/endpoint/server/routes/index_pattern.ts @@ -6,25 +6,21 @@ import { IRouter, Logger, RequestHandler } from 'kibana/server'; import { EndpointAppContext } from '../types'; -import { IndexPatternGetParamsResult, IndexPatternGetQueryParamsResult } from '../../common/types'; -import { - indexPatternGetParamsSchema, - indexPatternGetQueryParamsSchema, -} from '../../common/schema/index_pattern'; +import { IndexPatternGetParamsResult } from '../../common/types'; +import { indexPatternGetParamsSchema } from '../../common/schema/index_pattern'; import { IndexPatternRetriever } from '../index_pattern'; function handleIndexPattern( log: Logger, indexRetriever: IndexPatternRetriever -): RequestHandler { +): RequestHandler { return async (context, req, res) => { try { return res.ok({ body: { indexPattern: await indexRetriever.get( context.core.savedObjects.client, - req.params.datasetPath, - req.query.version + req.params.datasetPath ), }, }); @@ -41,7 +37,7 @@ export function registerIndexPatternRoute(router: IRouter, endpointAppContext: E router.get( { path: '/api/endpoint/index_pattern/{datasetPath}', - validate: { params: indexPatternGetParamsSchema, query: indexPatternGetQueryParamsSchema }, + validate: { params: indexPatternGetParamsSchema }, options: { authRequired: true }, }, handleIndexPattern(log, endpointAppContext.indexPatternRetriever) diff --git a/x-pack/plugins/ingest_manager/server/services/index_pattern.ts b/x-pack/plugins/ingest_manager/server/services/index_pattern.ts index c1a881eaf36cb..8cd0fb12d9a67 100644 --- a/x-pack/plugins/ingest_manager/server/services/index_pattern.ts +++ b/x-pack/plugins/ingest_manager/server/services/index_pattern.ts @@ -4,16 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ import { SavedObjectsClientContract } from 'kibana/server'; -import * as Registry from './epm/registry'; -import { getInstallationObject, findInstalledPackageByName } from './epm/packages/get'; -import { Installation } from '../types'; +import { getInstallation } from './epm/packages/get'; export interface IndexPatternService { get( savedObjectsClient: SavedObjectsClientContract, pkgName: string, - datasetPath: string, - version?: string + datasetPath: string ): Promise; } @@ -21,19 +18,9 @@ export class IndexPatternSavedObjectService implements IndexPatternService { public async get( savedObjectsClient: SavedObjectsClientContract, pkgName: string, - datasetPath: string, - version?: string + datasetPath: string ): Promise { - let installation: Installation | undefined; - if (version) { - const pkgkey = Registry.pkgToPkgKey({ - name: pkgName, - version, - }); - installation = (await getInstallationObject({ savedObjectsClient, pkgkey }))?.attributes; - } else { - installation = await findInstalledPackageByName({ savedObjectsClient, pkgName }); - } + const installation = await getInstallation({ savedObjectsClient, pkgName }); return installation?.installed.patterns[datasetPath]; } } diff --git a/x-pack/test/api_integration/apis/endpoint/index_pattern.ts b/x-pack/test/api_integration/apis/endpoint/index_pattern.ts index 530f542a41d99..d3ffd67defef1 100644 --- a/x-pack/test/api_integration/apis/endpoint/index_pattern.ts +++ b/x-pack/test/api_integration/apis/endpoint/index_pattern.ts @@ -11,26 +11,17 @@ export default function({ getService }: FtrProviderContext) { describe('Endpoint index pattern API', () => { it('should retrieve the index pattern for events', async () => { - const { body } = await supertest - .get('/api/endpoint/index_pattern/events') - .set('kbn-xsrf', 'xxx') - .expect(200); + const { body } = await supertest.get('/api/endpoint/index_pattern/events').expect(200); expect(body.indexPattern).to.eql('events-endpoint-*'); }); it('should retrieve the index pattern for metadata', async () => { - const { body } = await supertest - .get('/api/endpoint/index_pattern/metadata') - .set('kbn-xsrf', 'xxx') - .expect(200); + const { body } = await supertest.get('/api/endpoint/index_pattern/metadata').expect(200); expect(body.indexPattern).to.eql('metrics-endpoint-*'); }); it('should not retrieve the index pattern for an invalid key', async () => { - await supertest - .get('/api/endpoint/index_pattern/blah') - .set('kbn-xsrf', 'xxx') - .expect(404); + await supertest.get('/api/endpoint/index_pattern/blah').expect(404); }); }); } diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/alerts.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/alerts.ts index eb50f3cc0ab06..b75d69238d653 100644 --- a/x-pack/test/endpoint_api_integration_no_ingest/apis/alerts.ts +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/alerts.ts @@ -20,11 +20,8 @@ export default function({ getService }: FtrProviderContext) { await esArchiver.unload('endpoint/alerts/host_api_feature'); }); - it('should return zero results', async () => { - await supertest - .get('/api/endpoint/alerts') - .set('kbn-xsrf', 'xxx') - .expect(500); + it('should return a 500', async () => { + await supertest.get('/api/endpoint/alerts').expect(500); }); }); } diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/index_pattern.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/index_pattern.ts index b0934e4da7da0..664ef7d96847c 100644 --- a/x-pack/test/endpoint_api_integration_no_ingest/apis/index_pattern.ts +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/index_pattern.ts @@ -9,11 +9,8 @@ export default function({ getService }: FtrProviderContext) { const supertest = getService('supertest'); describe('Endpoint index pattern API without ingest manager initialized', () => { - it('should retrieve the index pattern for events', async () => { - await supertest - .get('/api/endpoint/index_pattern/events') - .set('kbn-xsrf', 'xxx') - .expect(404); + it('should not retrieve the index pattern for events', async () => { + await supertest.get('/api/endpoint/index_pattern/events').expect(404); }); }); } diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/metadata.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/metadata.ts index 15e9de5da8234..886d3cf3d9516 100644 --- a/x-pack/test/endpoint_api_integration_no_ingest/apis/metadata.ts +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/metadata.ts @@ -11,7 +11,7 @@ export default function({ getService }: FtrProviderContext) { describe('test metadata api when ingest manager is not initialized', () => { before(async () => await esArchiver.load('endpoint/metadata/api_feature')); after(async () => await esArchiver.unload('endpoint/metadata/api_feature')); - it('metadata api should return no results', async () => { + it('metadata api should return a 500', async () => { await supertest .post('/api/endpoint/metadata') .set('kbn-xsrf', 'xxx') From e94629f19ba7310150b61bcee1ae566c946ba40d Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 14 Apr 2020 15:55:02 -0400 Subject: [PATCH 18/27] Addressing endpoint index pattern feedback --- x-pack/plugins/endpoint/common/types.ts | 2 +- .../endpoint/store/alerts/middleware.ts | 2 +- .../plugins/endpoint/server/index_pattern.ts | 32 +++++++++++++----- x-pack/plugins/endpoint/server/mocks.ts | 28 ++++++++++++++++ x-pack/plugins/endpoint/server/plugin.test.ts | 33 ++----------------- .../server/routes/alerts/alerts.test.ts | 5 ++- .../endpoint/server/routes/index_pattern.ts | 9 ++--- .../endpoint/server/routes/metadata/index.ts | 10 +++--- .../server/routes/metadata/metadata.test.ts | 4 +-- .../routes/metadata/query_builders.test.ts | 6 ++-- 10 files changed, 70 insertions(+), 61 deletions(-) create mode 100644 x-pack/plugins/endpoint/server/mocks.ts diff --git a/x-pack/plugins/endpoint/common/types.ts b/x-pack/plugins/endpoint/common/types.ts index 617ed7613eadd..1bbd7b67930b2 100644 --- a/x-pack/plugins/endpoint/common/types.ts +++ b/x-pack/plugins/endpoint/common/types.ts @@ -31,9 +31,9 @@ export type Direction = 'asc' | 'desc'; export class EndpointAppConstants { static BASE_API_URL = '/api/endpoint'; + static INDEX_PATTERN_ROUTE = `${EndpointAppConstants.BASE_API_URL}/index_pattern`; static ALERT_INDEX_NAME = 'events-endpoint-1'; static EVENT_DATASET = 'events'; - static METADATA_DATASET = 'metadata'; static DEFAULT_TOTAL_HITS = 10000; /** * Legacy events are stored in indices with endgame-* prefix diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts b/x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts index bd2c695d9360f..52d72c6631443 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts +++ b/x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts @@ -16,7 +16,7 @@ export const alertMiddlewareFactory: MiddlewareFactory = (coreSt async function fetchIndexPatterns(): Promise { const { indexPatterns } = depsStart.data; const eventsPattern: { indexPattern: string } = await coreStart.http.get( - `/api/endpoint/index_pattern/${EndpointAppConstants.EVENT_DATASET}` + `${EndpointAppConstants.INDEX_PATTERN_ROUTE}/${EndpointAppConstants.EVENT_DATASET}` ); const fields = await indexPatterns.getFieldsForWildcard({ pattern: eventsPattern.indexPattern, diff --git a/x-pack/plugins/endpoint/server/index_pattern.ts b/x-pack/plugins/endpoint/server/index_pattern.ts index 0dc5b928a96f1..59f054bfa38a0 100644 --- a/x-pack/plugins/endpoint/server/index_pattern.ts +++ b/x-pack/plugins/endpoint/server/index_pattern.ts @@ -3,27 +3,43 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { SavedObjectsClientContract, Logger, LoggerFactory } from 'kibana/server'; +import { Logger, LoggerFactory, RequestHandlerContext } from 'kibana/server'; import { IndexPatternService } from '../../ingest_manager/server'; - -const endpointPackageName = 'endpoint'; +import { EndpointAppConstants } from '../common/types'; export interface IndexPatternRetriever { - get(client: SavedObjectsClientContract, datasetPath: string): Promise; + getIndexPattern(ctx: RequestHandlerContext, datasetPath: string): Promise; + getEventIndexPattern(ctx: RequestHandlerContext): Promise; + getMetadataIndexPattern(ctx: RequestHandlerContext): Promise; } export class IngestIndexPatternRetriever implements IndexPatternRetriever { + private static endpointPackageName = 'endpoint'; + private static metadataDataset = 'metadata'; private readonly log: Logger; constructor(private readonly service: IndexPatternService, loggerFactory: LoggerFactory) { this.log = loggerFactory.get('index-pattern-retriever'); } - async get(client: SavedObjectsClientContract, datasetPath: string, version?: string) { - const pattern = await this.service.get(client, endpointPackageName, datasetPath); + async getEventIndexPattern(ctx: RequestHandlerContext) { + return await this.getIndexPattern(ctx, EndpointAppConstants.EVENT_DATASET); + } + + async getMetadataIndexPattern(ctx: RequestHandlerContext) { + return await this.getIndexPattern(ctx, IngestIndexPatternRetriever.metadataDataset); + } + + async getIndexPattern(ctx: RequestHandlerContext, datasetPath: string) { + const pattern = await this.service.get( + ctx.core.savedObjects.client, + IngestIndexPatternRetriever.endpointPackageName, + datasetPath + ); if (!pattern) { - this.log.warn(`Failed to retrieve index pattern from ingest manager dataset: ${datasetPath}`); - throw new Error(`Unable to retrieve the index pattern for dataset: ${datasetPath}`); + const msg = `Unable to retrieve the index pattern for dataset: ${datasetPath}`; + this.log.warn(msg); + throw new Error(msg); } return pattern; diff --git a/x-pack/plugins/endpoint/server/mocks.ts b/x-pack/plugins/endpoint/server/mocks.ts new file mode 100644 index 0000000000000..ae2b025ff1251 --- /dev/null +++ b/x-pack/plugins/endpoint/server/mocks.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const createMockIndexPatternRetriever = (indexPattern: string) => { + const mockGetFunc = jest.fn().mockResolvedValue(indexPattern); + return { + getIndexPattern: mockGetFunc, + getEventIndexPattern: mockGetFunc, + getMetadataIndexPattern: mockGetFunc, + }; +}; + +export const MetadataIndexPattern = 'metrics-endpoint-*'; + +export const createMockMetadataIPRetriever = () => { + return createMockIndexPatternRetriever(MetadataIndexPattern); +}; + +export const createMockIndexPatternService = (indexPattern: string) => { + return { + indexPatternService: { + get: jest.fn().mockResolvedValue(indexPattern), + }, + }; +}; diff --git a/x-pack/plugins/endpoint/server/plugin.test.ts b/x-pack/plugins/endpoint/server/plugin.test.ts index 8e889ed0f1601..8d55e64f16dcf 100644 --- a/x-pack/plugins/endpoint/server/plugin.test.ts +++ b/x-pack/plugins/endpoint/server/plugin.test.ts @@ -3,40 +3,11 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -/* eslint-disable max-classes-per-file */ -import { SavedObjectsClientContract } from 'kibana/server'; import { EndpointPlugin, EndpointPluginSetupDependencies } from './plugin'; import { coreMock } from '../../../../src/core/server/mocks'; import { PluginSetupContract } from '../../features/server'; -import { IndexPatternService } from '../../ingest_manager/server'; -import { IndexPatternRetriever } from './index_pattern'; - -export const MetadataIndexPattern = 'metadata-endpoint-*'; - -export class FakeIndexPatternRetriever implements IndexPatternRetriever { - constructor(private readonly indexPattern: string) {} - - static buildMetadata() { - return new FakeIndexPatternRetriever(MetadataIndexPattern); - } - - async get(client: SavedObjectsClientContract, datasetPath: string) { - return this.indexPattern; - } -} - -export class FakeIndexPatternService implements IndexPatternService { - constructor(private readonly indexPattern: string) {} - - async get( - savedObjectsClient: SavedObjectsClientContract, - pkgName: string, - datasetPath: string - ): Promise { - return this.indexPattern; - } -} +import { createMockIndexPatternService } from './mocks'; describe('test endpoint plugin', () => { let plugin: EndpointPlugin; @@ -60,7 +31,7 @@ describe('test endpoint plugin', () => { }; mockedEndpointPluginSetupDependencies = { features: mockedPluginSetupContract, - ingestManager: { indexPatternService: new FakeIndexPatternService('') }, + ingestManager: createMockIndexPatternService(''), }; }); diff --git a/x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts b/x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts index e245914360521..6be7b26898206 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts @@ -12,7 +12,7 @@ import { import { registerAlertRoutes } from './index'; import { EndpointConfigSchema } from '../../config'; import { alertingIndexGetQuerySchema } from '../../../common/schema/alert_index'; -import { FakeIndexPatternRetriever } from '../../plugin.test'; +import { createMockIndexPatternRetriever } from '../../mocks'; describe('test alerts route', () => { let routerMock: jest.Mocked; @@ -25,8 +25,7 @@ describe('test alerts route', () => { mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); registerAlertRoutes(routerMock, { - indexPatternRetriever: new FakeIndexPatternRetriever('events-endpoint-*'), - + indexPatternRetriever: createMockIndexPatternRetriever('events-endpoint-*'), logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }); diff --git a/x-pack/plugins/endpoint/server/routes/index_pattern.ts b/x-pack/plugins/endpoint/server/routes/index_pattern.ts index 69ff27a9d003d..3b71f6a6957ba 100644 --- a/x-pack/plugins/endpoint/server/routes/index_pattern.ts +++ b/x-pack/plugins/endpoint/server/routes/index_pattern.ts @@ -6,7 +6,7 @@ import { IRouter, Logger, RequestHandler } from 'kibana/server'; import { EndpointAppContext } from '../types'; -import { IndexPatternGetParamsResult } from '../../common/types'; +import { IndexPatternGetParamsResult, EndpointAppConstants } from '../../common/types'; import { indexPatternGetParamsSchema } from '../../common/schema/index_pattern'; import { IndexPatternRetriever } from '../index_pattern'; @@ -18,10 +18,7 @@ function handleIndexPattern( try { return res.ok({ body: { - indexPattern: await indexRetriever.get( - context.core.savedObjects.client, - req.params.datasetPath - ), + indexPattern: await indexRetriever.getIndexPattern(context, req.params.datasetPath), }, }); } catch (error) { @@ -36,7 +33,7 @@ export function registerIndexPatternRoute(router: IRouter, endpointAppContext: E router.get( { - path: '/api/endpoint/index_pattern/{datasetPath}', + path: `${EndpointAppConstants.INDEX_PATTERN_ROUTE}/{datasetPath}`, validate: { params: indexPatternGetParamsSchema }, options: { authRequired: true }, }, diff --git a/x-pack/plugins/endpoint/server/routes/metadata/index.ts b/x-pack/plugins/endpoint/server/routes/metadata/index.ts index 63a2e25cc7e90..1be27a529fb16 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/index.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/index.ts @@ -56,9 +56,8 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp }, async (context, req, res) => { try { - const index = await endpointAppContext.indexPatternRetriever.get( - context.core.savedObjects.client, - EndpointAppConstants.METADATA_DATASET + const index = await endpointAppContext.indexPatternRetriever.getMetadataIndexPattern( + context ); const queryParams = await kibanaRequestToMetadataListESQuery( req, @@ -86,9 +85,8 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp }, async (context, req, res) => { try { - const index = await endpointAppContext.indexPatternRetriever.get( - context.core.savedObjects.client, - EndpointAppConstants.METADATA_DATASET + const index = await endpointAppContext.indexPatternRetriever.getMetadataIndexPattern( + context ); const doc = await getHostData(context, req.params.id, index); diff --git a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts index 6d1be57422acc..227003e3908a4 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts @@ -24,8 +24,8 @@ import { HostInfo, HostMetadata, HostResultList, HostStatus } from '../../../com import { SearchResponse } from 'elasticsearch'; import { registerEndpointRoutes } from './index'; import { EndpointConfigSchema } from '../../config'; -import { FakeIndexPatternRetriever } from '../../plugin.test'; import * as data from '../../test_data/all_metadata_data.json'; +import { createMockMetadataIPRetriever } from '../../mocks'; describe('test endpoint route', () => { let routerMock: jest.Mocked; @@ -46,7 +46,7 @@ describe('test endpoint route', () => { routerMock = httpServiceMock.createRouter(); mockResponse = httpServerMock.createResponseFactory(); registerEndpointRoutes(routerMock, { - indexPatternRetriever: FakeIndexPatternRetriever.buildMetadata(), + indexPatternRetriever: createMockMetadataIPRetriever(), logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }); diff --git a/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts b/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts index cb17b7bc000c9..8229f6848bcd9 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts @@ -6,7 +6,7 @@ import { httpServerMock, loggingServiceMock } from '../../../../../../src/core/server/mocks'; import { EndpointConfigSchema } from '../../config'; import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders'; -import { MetadataIndexPattern, FakeIndexPatternRetriever } from '../../plugin.test'; +import { createMockMetadataIPRetriever, MetadataIndexPattern } from '../../mocks'; describe('query builder', () => { describe('MetadataListESQuery', () => { @@ -17,7 +17,7 @@ describe('query builder', () => { const query = await kibanaRequestToMetadataListESQuery( mockRequest, { - indexPatternRetriever: FakeIndexPatternRetriever.buildMetadata(), + indexPatternRetriever: createMockMetadataIPRetriever(), logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }, @@ -68,7 +68,7 @@ describe('query builder', () => { const query = await kibanaRequestToMetadataListESQuery( mockRequest, { - indexPatternRetriever: FakeIndexPatternRetriever.buildMetadata(), + indexPatternRetriever: createMockMetadataIPRetriever(), logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }, From 89dd8228f50ef032e5a2d4ed5ff4533a68b34ddd Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 14 Apr 2020 15:55:45 -0400 Subject: [PATCH 19/27] Removing unused import --- x-pack/plugins/endpoint/server/routes/metadata/index.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/x-pack/plugins/endpoint/server/routes/metadata/index.ts b/x-pack/plugins/endpoint/server/routes/metadata/index.ts index 1be27a529fb16..883bb88204fd4 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/index.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/index.ts @@ -9,13 +9,7 @@ import { SearchResponse } from 'elasticsearch'; import { schema } from '@kbn/config-schema'; import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders'; -import { - HostInfo, - HostMetadata, - HostResultList, - HostStatus, - EndpointAppConstants, -} from '../../../common/types'; +import { HostInfo, HostMetadata, HostResultList, HostStatus } from '../../../common/types'; import { EndpointAppContext } from '../../types'; interface HitSource { From 337ab45681cfe9e7f717fe57ea7731b2a6ff22c0 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 14 Apr 2020 16:28:44 -0400 Subject: [PATCH 20/27] Renaming index pattern to es index pattern --- .../plugins/endpoint/server/index_pattern.ts | 6 ++--- x-pack/plugins/endpoint/server/mocks.ts | 4 ++-- x-pack/plugins/endpoint/server/plugin.ts | 2 +- .../ingest_manager/common/types/models/epm.ts | 2 +- x-pack/plugins/ingest_manager/server/index.ts | 2 +- .../plugins/ingest_manager/server/plugin.ts | 10 +++++--- .../ingest_manager/server/saved_objects.ts | 2 +- .../epm/elasticsearch/template/template.ts | 4 ++-- .../server/services/epm/packages/install.ts | 24 +++++++++++++------ .../{index_pattern.ts => es_index_pattern.ts} | 10 ++++---- .../ingest_manager/server/services/index.ts | 2 +- 11 files changed, 41 insertions(+), 27 deletions(-) rename x-pack/plugins/ingest_manager/server/services/{index_pattern.ts => es_index_pattern.ts} (74%) diff --git a/x-pack/plugins/endpoint/server/index_pattern.ts b/x-pack/plugins/endpoint/server/index_pattern.ts index 59f054bfa38a0..cc1fe347ac65a 100644 --- a/x-pack/plugins/endpoint/server/index_pattern.ts +++ b/x-pack/plugins/endpoint/server/index_pattern.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { Logger, LoggerFactory, RequestHandlerContext } from 'kibana/server'; -import { IndexPatternService } from '../../ingest_manager/server'; +import { ESIndexPatternService } from '../../ingest_manager/server'; import { EndpointAppConstants } from '../common/types'; export interface IndexPatternRetriever { @@ -17,7 +17,7 @@ export class IngestIndexPatternRetriever implements IndexPatternRetriever { private static endpointPackageName = 'endpoint'; private static metadataDataset = 'metadata'; private readonly log: Logger; - constructor(private readonly service: IndexPatternService, loggerFactory: LoggerFactory) { + constructor(private readonly service: ESIndexPatternService, loggerFactory: LoggerFactory) { this.log = loggerFactory.get('index-pattern-retriever'); } @@ -30,7 +30,7 @@ export class IngestIndexPatternRetriever implements IndexPatternRetriever { } async getIndexPattern(ctx: RequestHandlerContext, datasetPath: string) { - const pattern = await this.service.get( + const pattern = await this.service.getESIndexPattern( ctx.core.savedObjects.client, IngestIndexPatternRetriever.endpointPackageName, datasetPath diff --git a/x-pack/plugins/endpoint/server/mocks.ts b/x-pack/plugins/endpoint/server/mocks.ts index ae2b025ff1251..029f4a8791ed7 100644 --- a/x-pack/plugins/endpoint/server/mocks.ts +++ b/x-pack/plugins/endpoint/server/mocks.ts @@ -21,8 +21,8 @@ export const createMockMetadataIPRetriever = () => { export const createMockIndexPatternService = (indexPattern: string) => { return { - indexPatternService: { - get: jest.fn().mockResolvedValue(indexPattern), + esIndexPatternService: { + getESIndexPattern: jest.fn().mockResolvedValue(indexPattern), }, }; }; diff --git a/x-pack/plugins/endpoint/server/plugin.ts b/x-pack/plugins/endpoint/server/plugin.ts index e4e41045a6c10..6a42014e91130 100644 --- a/x-pack/plugins/endpoint/server/plugin.ts +++ b/x-pack/plugins/endpoint/server/plugin.ts @@ -67,7 +67,7 @@ export class EndpointPlugin }); const endpointContext = { indexPatternRetriever: new IngestIndexPatternRetriever( - plugins.ingestManager.indexPatternService, + plugins.ingestManager.esIndexPatternService, this.initializerContext.logger ), logFactory: this.initializerContext.logger, diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index ddb5aa84a9cf8..51e914402f095 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -218,7 +218,7 @@ export type PackageInfo = Installable< export interface InstalledReferences extends SavedObjectAttributes { references: AssetReference[]; - patterns: Record; + es_index_patterns: Record; } export interface Installation extends SavedObjectAttributes { diff --git a/x-pack/plugins/ingest_manager/server/index.ts b/x-pack/plugins/ingest_manager/server/index.ts index 8bc6ae788f070..7859c44ccfd89 100644 --- a/x-pack/plugins/ingest_manager/server/index.ts +++ b/x-pack/plugins/ingest_manager/server/index.ts @@ -7,7 +7,7 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { PluginInitializerContext } from 'src/core/server'; import { IngestManagerPlugin } from './plugin'; -export { IndexPatternService } from './services'; +export { ESIndexPatternService } from './services'; export { IngestManagerSetupContract } from './plugin'; export const config = { diff --git a/x-pack/plugins/ingest_manager/server/plugin.ts b/x-pack/plugins/ingest_manager/server/plugin.ts index b37df701c4127..4dd070a7414f0 100644 --- a/x-pack/plugins/ingest_manager/server/plugin.ts +++ b/x-pack/plugins/ingest_manager/server/plugin.ts @@ -40,13 +40,17 @@ import { } from './routes'; import { IngestManagerConfigType } from '../common'; -import { appContextService, IndexPatternService, IndexPatternSavedObjectService } from './services'; +import { + appContextService, + ESIndexPatternService, + ESIndexPatternSavedObjectService, +} from './services'; /** * Describes public IngestManager plugin contract returned at the `setup` stage. */ export interface IngestManagerSetupContract { - indexPatternService: IndexPatternService; + esIndexPatternService: ESIndexPatternService; } export interface IngestManagerSetupDeps { @@ -143,7 +147,7 @@ export class IngestManagerPlugin implements Plugin { }); } return deepFreeze({ - indexPatternService: new IndexPatternSavedObjectService(), + esIndexPatternService: new ESIndexPatternSavedObjectService(), }); } diff --git a/x-pack/plugins/ingest_manager/server/saved_objects.ts b/x-pack/plugins/ingest_manager/server/saved_objects.ts index afbf27232af34..69129d5366bbc 100644 --- a/x-pack/plugins/ingest_manager/server/saved_objects.ts +++ b/x-pack/plugins/ingest_manager/server/saved_objects.ts @@ -158,7 +158,7 @@ export const savedObjectMappings = { type: { type: 'keyword' }, }, }, - patterns: { + es_index_patterns: { dynamic: false, type: 'object', }, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index b271cd30f07ee..832e4772beb0d 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -137,10 +137,10 @@ export function generateTemplateName(dataset: Dataset): string { } /** - * Returns a map of the dataset path fields to index pattern. + * Returns a map of the dataset path fields to elasticsearch index pattern. * @param datasets an array of Dataset objects */ -export function generateIndexPatterns(datasets: Dataset[] | undefined): Record { +export function generateESIndexPatterns(datasets: Dataset[] | undefined): Record { if (!datasets) { return {}; } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 731917f1f3c6f..a5bfc9a107287 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -19,7 +19,7 @@ import * as Registry from '../registry'; import { getObject } from './get_objects'; import { getInstallation } from './index'; import { installTemplates } from '../elasticsearch/template/install'; -import { generateIndexPatterns } from '../elasticsearch/template/template'; +import { generateESIndexPatterns } from '../elasticsearch/template/template'; import { installPipelines } from '../elasticsearch/ingest_pipeline/install'; import { installILMPolicy } from '../elasticsearch/ilm/install'; @@ -120,7 +120,7 @@ export async function installPackage(options: { ]); const toSaveRefs: AssetReference[] = res.flat(); - const toSavePatterns = generateIndexPatterns(registryPackageInfo.datasets); + const toSaveESIndexPatterns = generateESIndexPatterns(registryPackageInfo.datasets); // Save those references in the package manager's state saved object return await saveInstallationReferences({ savedObjectsClient, @@ -129,7 +129,7 @@ export async function installPackage(options: { pkgVersion, internal, toSaveRefs, - toSavePatterns, + toSaveESIndexPatterns, }); } @@ -160,12 +160,22 @@ export async function saveInstallationReferences(options: { pkgVersion: string; internal: boolean; toSaveRefs: AssetReference[]; - toSavePatterns: Record; + toSaveESIndexPatterns: Record; }) { - const { savedObjectsClient, pkgName, pkgVersion, internal, toSaveRefs, toSavePatterns } = options; + const { + savedObjectsClient, + pkgName, + pkgVersion, + internal, + toSaveRefs, + toSaveESIndexPatterns, + } = options; const installation = await getInstallation({ savedObjectsClient, pkgName }); const savedRefs = installation?.installed.references || []; - const toInstallStreams = Object.assign(installation?.datasetIndexPattern || {}, toSavePatterns); + const toInstallESIndexPatterns = Object.assign( + installation?.installed.es_index_patterns || {}, + toSaveESIndexPatterns + ); const mergeRefsReducer = (current: AssetReference[], pending: AssetReference) => { const hasRef = current.find(c => c.id === pending.id && c.type === pending.type); @@ -174,7 +184,7 @@ export async function saveInstallationReferences(options: { }; const toInstallRefs = toSaveRefs.reduce(mergeRefsReducer, savedRefs); - const installed = { references: toInstallRefs, patterns: toInstallStreams }; + const installed = { references: toInstallRefs, es_index_patterns: toInstallESIndexPatterns }; await savedObjectsClient.create( PACKAGES_SAVED_OBJECT_TYPE, { diff --git a/x-pack/plugins/ingest_manager/server/services/index_pattern.ts b/x-pack/plugins/ingest_manager/server/services/es_index_pattern.ts similarity index 74% rename from x-pack/plugins/ingest_manager/server/services/index_pattern.ts rename to x-pack/plugins/ingest_manager/server/services/es_index_pattern.ts index 8cd0fb12d9a67..d6a394f6abfca 100644 --- a/x-pack/plugins/ingest_manager/server/services/index_pattern.ts +++ b/x-pack/plugins/ingest_manager/server/services/es_index_pattern.ts @@ -6,21 +6,21 @@ import { SavedObjectsClientContract } from 'kibana/server'; import { getInstallation } from './epm/packages/get'; -export interface IndexPatternService { - get( +export interface ESIndexPatternService { + getESIndexPattern( savedObjectsClient: SavedObjectsClientContract, pkgName: string, datasetPath: string ): Promise; } -export class IndexPatternSavedObjectService implements IndexPatternService { - public async get( +export class ESIndexPatternSavedObjectService implements ESIndexPatternService { + public async getESIndexPattern( savedObjectsClient: SavedObjectsClientContract, pkgName: string, datasetPath: string ): Promise { const installation = await getInstallation({ savedObjectsClient, pkgName }); - return installation?.installed.patterns[datasetPath]; + return installation?.installed.es_index_patterns[datasetPath]; } } diff --git a/x-pack/plugins/ingest_manager/server/services/index.ts b/x-pack/plugins/ingest_manager/server/services/index.ts index 0b5ef90572524..d64f1b0c2b6fb 100644 --- a/x-pack/plugins/ingest_manager/server/services/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/index.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ export { appContextService } from './app_context'; -export { IndexPatternService, IndexPatternSavedObjectService } from './index_pattern'; +export { ESIndexPatternService, ESIndexPatternSavedObjectService } from './es_index_pattern'; // Saved object services export { datasourceService } from './datasource'; From 4dc5854ce48b26446bc4d9d269bcea389a7d174f Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 14 Apr 2020 16:48:45 -0400 Subject: [PATCH 21/27] Fixing missed index pattern calls --- .../endpoint/server/routes/alerts/details/handlers.ts | 5 +---- .../plugins/endpoint/server/routes/alerts/list/handlers.ts | 5 +---- x-pack/plugins/endpoint/server/routes/resolver/children.ts | 5 +---- x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts | 5 +---- .../endpoint/server/routes/resolver/related_events.ts | 5 +---- 5 files changed, 5 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts b/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts index 35640d8ed9e8f..8ed673d20bb9c 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts @@ -28,10 +28,7 @@ export const alertDetailsHandlerWrapper = function( id: alertId, })) as GetResponse; - const indexPattern = await endpointAppContext.indexPatternRetriever.get( - ctx.core.savedObjects.client, - EndpointAppConstants.EVENT_DATASET - ); + const indexPattern = await endpointAppContext.indexPatternRetriever.getEventIndexPattern(ctx); const config = await endpointAppContext.config(); const pagination: AlertDetailsPagination = new AlertDetailsPagination( diff --git a/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts b/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts index 8441034c35edf..ef7da3b864068 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts @@ -18,10 +18,7 @@ export const alertListHandlerWrapper = function( res ) => { try { - const indexPattern = await endpointAppContext.indexPatternRetriever.get( - ctx.core.savedObjects.client, - EndpointAppConstants.EVENT_DATASET - ); + const indexPattern = await endpointAppContext.indexPatternRetriever.getEventIndexPattern(ctx); const reqData = await getRequestData(req, endpointAppContext); const response = await searchESForAlerts( ctx.core.elasticsearch.dataClient, diff --git a/x-pack/plugins/endpoint/server/routes/resolver/children.ts b/x-pack/plugins/endpoint/server/routes/resolver/children.ts index b2f3edfcba784..d4a02be57cd6f 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/children.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/children.ts @@ -57,10 +57,7 @@ export function handleChildren( } = req; try { const pagination = getPaginationParams(limit, after); - const indexPattern = await indexRetriever.get( - context.core.savedObjects.client, - EndpointAppConstants.EVENT_DATASET - ); + const indexPattern = await indexRetriever.getEventIndexPattern(context); const client = context.core.elasticsearch.dataClient; const childrenQuery = new ChildrenQuery(indexPattern, legacyEndpointID, pagination); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts b/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts index 0da1cfdb32761..c1c4aa0cd0d0f 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts @@ -59,10 +59,7 @@ export function handleLifecycle( try { const ancestorLifecycles = []; const client = context.core.elasticsearch.dataClient; - const indexPattern = await indexRetriever.get( - context.core.savedObjects.client, - EndpointAppConstants.EVENT_DATASET - ); + const indexPattern = await indexRetriever.getEventIndexPattern(context); const lifecycleQuery = new LifecycleQuery(indexPattern, legacyEndpointID); const { results: processLifecycle } = await lifecycleQuery.search(client, id); let nextParentID = getParentEntityID(processLifecycle); diff --git a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts b/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts index d6e9dd586e86c..1bfafb7822048 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts @@ -56,10 +56,7 @@ export function handleRelatedEvents( const pagination = getPaginationParams(limit, after); const client = context.core.elasticsearch.dataClient; - const indexPattern = await indexRetriever.get( - context.core.savedObjects.client, - EndpointAppConstants.EVENT_DATASET - ); + const indexPattern = await indexRetriever.getEventIndexPattern(context); // Retrieve the related non-process events for a given process const relatedEventsQuery = new RelatedEventsQuery(indexPattern, legacyEndpointID, pagination); const relatedEvents = await relatedEventsQuery.search(client, id); From b66929434e6c6712a60319057aefb84fae9fc518 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 14 Apr 2020 16:51:14 -0400 Subject: [PATCH 22/27] Removing unused import --- x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts | 2 +- x-pack/plugins/endpoint/server/routes/resolver/children.ts | 1 - x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts | 1 - .../plugins/endpoint/server/routes/resolver/related_events.ts | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts b/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts index ef7da3b864068..f23dffd13db4f 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts @@ -7,7 +7,7 @@ import { RequestHandler } from 'kibana/server'; import { EndpointAppContext } from '../../../types'; import { searchESForAlerts } from '../lib'; import { getRequestData, mapToAlertResultList } from './lib'; -import { AlertingIndexGetQueryResult, EndpointAppConstants } from '../../../../common/types'; +import { AlertingIndexGetQueryResult } from '../../../../common/types'; export const alertListHandlerWrapper = function( endpointAppContext: EndpointAppContext diff --git a/x-pack/plugins/endpoint/server/routes/resolver/children.ts b/x-pack/plugins/endpoint/server/routes/resolver/children.ts index d4a02be57cd6f..05b8f0b5f8608 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/children.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/children.ts @@ -11,7 +11,6 @@ import { extractEntityID } from './utils/normalize'; import { getPaginationParams } from './utils/pagination'; import { LifecycleQuery } from './queries/lifecycle'; import { ChildrenQuery } from './queries/children'; -import { EndpointAppConstants } from '../../../common/types'; import { IndexPatternRetriever } from '../../index_pattern'; interface ChildrenQueryParams { diff --git a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts b/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts index c1c4aa0cd0d0f..6d155b79651a7 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts @@ -10,7 +10,6 @@ import { RequestHandler, Logger } from 'kibana/server'; import { extractParentEntityID } from './utils/normalize'; import { LifecycleQuery } from './queries/lifecycle'; import { ResolverEvent } from '../../../common/types'; -import { EndpointAppConstants } from '../../../common/types'; import { IndexPatternRetriever } from '../../index_pattern'; interface LifecycleQueryParams { diff --git a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts b/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts index 1bfafb7822048..46e205464f53c 100644 --- a/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts +++ b/x-pack/plugins/endpoint/server/routes/resolver/related_events.ts @@ -6,7 +6,6 @@ import { schema } from '@kbn/config-schema'; import { RequestHandler, Logger } from 'kibana/server'; -import { EndpointAppConstants } from '../../../common/types'; import { getPaginationParams } from './utils/pagination'; import { RelatedEventsQuery } from './queries/related_events'; import { IndexPatternRetriever } from '../../index_pattern'; From cc4a3551ccee2426f846752bd15adb4985e6e136 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 14 Apr 2020 17:39:03 -0400 Subject: [PATCH 23/27] Fixing type error --- .../ingest_manager/server/services/epm/packages/remove.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts index 1e95cfad4fb0f..42df66a20442a 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts @@ -20,7 +20,7 @@ export async function removeInstallation(options: { // TODO: the epm api should change to /name/version so we don't need to do this const [pkgName] = pkgkey.split('-'); const installation = await getInstallation({ savedObjectsClient, pkgName }); - const installedObjects = installation?.installed || { references: [], patterns: {} }; + const installedObjects = installation?.installed || { references: [], es_index_patterns: {} }; // Delete the manager saved object with references to the asset objects // could also update with [] or some other state From 1aa938897f8d84e62d0a59fc848acf69695ae720 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 15 Apr 2020 10:42:22 -0400 Subject: [PATCH 24/27] Moving es_index_pattern outside of installed and fixing function name --- x-pack/plugins/endpoint/server/mocks.ts | 2 +- .../server/routes/metadata/metadata.test.ts | 4 +-- .../routes/metadata/query_builders.test.ts | 6 ++--- .../ingest_manager/common/types/models/epm.ts | 8 ++---- .../server/routes/epm/handlers.ts | 2 +- .../ingest_manager/server/saved_objects.ts | 18 ++++++------- .../server/services/epm/packages/install.ts | 25 +++++++++---------- .../server/services/epm/packages/remove.ts | 6 ++--- .../server/services/es_index_pattern.ts | 2 +- .../ingest_manager/server/types/index.tsx | 1 - 10 files changed, 32 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/endpoint/server/mocks.ts b/x-pack/plugins/endpoint/server/mocks.ts index 029f4a8791ed7..5c8aba0678577 100644 --- a/x-pack/plugins/endpoint/server/mocks.ts +++ b/x-pack/plugins/endpoint/server/mocks.ts @@ -15,7 +15,7 @@ export const createMockIndexPatternRetriever = (indexPattern: string) => { export const MetadataIndexPattern = 'metrics-endpoint-*'; -export const createMockMetadataIPRetriever = () => { +export const createMockMetadataIndexPatternRetriever = () => { return createMockIndexPatternRetriever(MetadataIndexPattern); }; diff --git a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts index 227003e3908a4..b38339b5f1475 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts @@ -25,7 +25,7 @@ import { SearchResponse } from 'elasticsearch'; import { registerEndpointRoutes } from './index'; import { EndpointConfigSchema } from '../../config'; import * as data from '../../test_data/all_metadata_data.json'; -import { createMockMetadataIPRetriever } from '../../mocks'; +import { createMockMetadataIndexPatternRetriever } from '../../mocks'; describe('test endpoint route', () => { let routerMock: jest.Mocked; @@ -46,7 +46,7 @@ describe('test endpoint route', () => { routerMock = httpServiceMock.createRouter(); mockResponse = httpServerMock.createResponseFactory(); registerEndpointRoutes(routerMock, { - indexPatternRetriever: createMockMetadataIPRetriever(), + indexPatternRetriever: createMockMetadataIndexPatternRetriever(), logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }); diff --git a/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts b/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts index 8229f6848bcd9..c8143fbdda1ea 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/query_builders.test.ts @@ -6,7 +6,7 @@ import { httpServerMock, loggingServiceMock } from '../../../../../../src/core/server/mocks'; import { EndpointConfigSchema } from '../../config'; import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders'; -import { createMockMetadataIPRetriever, MetadataIndexPattern } from '../../mocks'; +import { createMockMetadataIndexPatternRetriever, MetadataIndexPattern } from '../../mocks'; describe('query builder', () => { describe('MetadataListESQuery', () => { @@ -17,7 +17,7 @@ describe('query builder', () => { const query = await kibanaRequestToMetadataListESQuery( mockRequest, { - indexPatternRetriever: createMockMetadataIPRetriever(), + indexPatternRetriever: createMockMetadataIndexPatternRetriever(), logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }, @@ -68,7 +68,7 @@ describe('query builder', () => { const query = await kibanaRequestToMetadataListESQuery( mockRequest, { - indexPatternRetriever: createMockMetadataIPRetriever(), + indexPatternRetriever: createMockMetadataIndexPatternRetriever(), logFactory: loggingServiceMock.create(), config: () => Promise.resolve(EndpointConfigSchema.validate({})), }, diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index deebc37bc4fb6..53ad0310ea613 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -216,13 +216,9 @@ export type PackageInfo = Installable< PackageAdditions >; -export interface InstalledReferences extends SavedObjectAttributes { - references: AssetReference[]; - es_index_patterns: Record; -} - export interface Installation extends SavedObjectAttributes { - installed: InstalledReferences; + installed: AssetReference[]; + es_index_patterns: Record; name: string; version: string; } diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index 7ff881601a7c0..ad16e1dde456b 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -131,7 +131,7 @@ export const installPackageHandler: RequestHandler { +}): Promise { const { savedObjectsClient, pkgName, callCluster } = options; try { const latestPackage = await Registry.fetchFindLatestPackage(pkgName); @@ -86,7 +85,7 @@ export async function installPackage(options: { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; callCluster: CallESAsCurrentUser; -}): Promise { +}): Promise { const { savedObjectsClient, pkgkey, callCluster } = options; // TODO: change epm API to /packageName/version so we don't need to do this const [pkgName, pkgVersion] = pkgkey.split('-'); @@ -119,7 +118,7 @@ export async function installPackage(options: { installTemplatePromises, ]); - const toSaveRefs: AssetReference[] = res.flat(); + const toSaveAssetRefs: AssetReference[] = res.flat(); const toSaveESIndexPatterns = generateESIndexPatterns(registryPackageInfo.datasets); // Save those references in the package manager's state saved object return await saveInstallationReferences({ @@ -128,7 +127,7 @@ export async function installPackage(options: { pkgName, pkgVersion, internal, - toSaveRefs, + toSaveAssetRefs, toSaveESIndexPatterns, }); } @@ -159,7 +158,7 @@ export async function saveInstallationReferences(options: { pkgName: string; pkgVersion: string; internal: boolean; - toSaveRefs: AssetReference[]; + toSaveAssetRefs: AssetReference[]; toSaveESIndexPatterns: Record; }) { const { @@ -167,13 +166,13 @@ export async function saveInstallationReferences(options: { pkgName, pkgVersion, internal, - toSaveRefs, + toSaveAssetRefs, toSaveESIndexPatterns, } = options; const installation = await getInstallation({ savedObjectsClient, pkgName }); - const savedRefs = installation?.installed.references || []; + const savedAssetRefs = installation?.installed || []; const toInstallESIndexPatterns = Object.assign( - installation?.installed.es_index_patterns || {}, + installation?.es_index_patterns || {}, toSaveESIndexPatterns ); @@ -183,12 +182,12 @@ export async function saveInstallationReferences(options: { return current; }; - const toInstallRefs = toSaveRefs.reduce(mergeRefsReducer, savedRefs); - const installed = { references: toInstallRefs, es_index_patterns: toInstallESIndexPatterns }; + const toInstallAssetsRefs = toSaveAssetRefs.reduce(mergeRefsReducer, savedAssetRefs); await savedObjectsClient.create( PACKAGES_SAVED_OBJECT_TYPE, { - installed, + installed: toInstallAssetsRefs, + es_index_patterns: toInstallESIndexPatterns, name: pkgName, version: pkgVersion, internal, @@ -196,7 +195,7 @@ export async function saveInstallationReferences(options: { { id: pkgName, overwrite: true } ); - return installed; + return toInstallAssetsRefs; } async function installKibanaSavedObjects({ diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts index 42df66a20442a..a30acb97b99cf 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts @@ -20,7 +20,7 @@ export async function removeInstallation(options: { // TODO: the epm api should change to /name/version so we don't need to do this const [pkgName] = pkgkey.split('-'); const installation = await getInstallation({ savedObjectsClient, pkgName }); - const installedObjects = installation?.installed || { references: [], es_index_patterns: {} }; + const installedObjects = installation?.installed || []; // Delete the manager saved object with references to the asset objects // could also update with [] or some other state @@ -30,7 +30,7 @@ export async function removeInstallation(options: { await installIndexPatterns(savedObjectsClient); // Delete the installed assets - const deletePromises = installedObjects.references.map(async ({ id, type }) => { + const deletePromises = installedObjects.map(async ({ id, type }) => { const assetType = type as AssetType; if (savedObjectTypes.includes(assetType)) { savedObjectsClient.delete(assetType, id); @@ -43,7 +43,7 @@ export async function removeInstallation(options: { await Promise.all([...deletePromises]); // successful delete's in SO client return {}. return something more useful - return installedObjects.references; + return installedObjects; } async function deletePipeline(callCluster: CallESAsCurrentUser, id: string): Promise { diff --git a/x-pack/plugins/ingest_manager/server/services/es_index_pattern.ts b/x-pack/plugins/ingest_manager/server/services/es_index_pattern.ts index d6a394f6abfca..167e22873979c 100644 --- a/x-pack/plugins/ingest_manager/server/services/es_index_pattern.ts +++ b/x-pack/plugins/ingest_manager/server/services/es_index_pattern.ts @@ -21,6 +21,6 @@ export class ESIndexPatternSavedObjectService implements ESIndexPatternService { datasetPath: string ): Promise { const installation = await getInstallation({ savedObjectsClient, pkgName }); - return installation?.installed.es_index_patterns[datasetPath]; + return installation?.es_index_patterns[datasetPath]; } } diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index aed94f75333ae..1cd5622c0c7b0 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -28,7 +28,6 @@ export { EnrollmentAPIKey, EnrollmentAPIKeySOAttributes, Installation, - InstalledReferences, InstallationStatus, PackageInfo, RegistryVarsEntry, From dd033d433860760092f9b2b8d4eaac8557d2e86c Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 15 Apr 2020 11:11:30 -0400 Subject: [PATCH 25/27] Keeping the event index the same for now --- x-pack/plugins/endpoint/scripts/resolver_generator.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/endpoint/scripts/resolver_generator.ts b/x-pack/plugins/endpoint/scripts/resolver_generator.ts index d409b42cfa261..0ecd756048d13 100644 --- a/x-pack/plugins/endpoint/scripts/resolver_generator.ts +++ b/x-pack/plugins/endpoint/scripts/resolver_generator.ts @@ -35,7 +35,9 @@ async function main() { eventIndex: { alias: 'ei', describe: 'index to store events in', - default: 'events-endpoint-default-1', + // This should be changed to events-endpoint-default-1 when this issue gets resolved: + // https://github.com/elastic/endpoint-app-team/issues/311 + default: 'events-endpoint-1', type: 'string', }, metadataIndex: { From efdcc2d3ebd3d222426b4a7287b698afad10ffe8 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Wed, 15 Apr 2020 11:58:40 -0400 Subject: [PATCH 26/27] Wrapping index pattern await in try catch --- .../plugins/endpoint/server/index_pattern.ts | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/endpoint/server/index_pattern.ts b/x-pack/plugins/endpoint/server/index_pattern.ts index cc1fe347ac65a..b35413598cc63 100644 --- a/x-pack/plugins/endpoint/server/index_pattern.ts +++ b/x-pack/plugins/endpoint/server/index_pattern.ts @@ -30,18 +30,23 @@ export class IngestIndexPatternRetriever implements IndexPatternRetriever { } async getIndexPattern(ctx: RequestHandlerContext, datasetPath: string) { - const pattern = await this.service.getESIndexPattern( - ctx.core.savedObjects.client, - IngestIndexPatternRetriever.endpointPackageName, - datasetPath - ); + try { + const pattern = await this.service.getESIndexPattern( + ctx.core.savedObjects.client, + IngestIndexPatternRetriever.endpointPackageName, + datasetPath + ); - if (!pattern) { - const msg = `Unable to retrieve the index pattern for dataset: ${datasetPath}`; - this.log.warn(msg); - throw new Error(msg); + if (!pattern) { + const msg = `Unable to retrieve the index pattern for dataset: ${datasetPath}`; + this.log.warn(msg); + throw new Error(msg); + } + return pattern; + } catch (error) { + const errMsg = `Error occurred while retrieving pattern for: ${datasetPath} error: ${error}`; + this.log.warn(errMsg); + throw new Error(errMsg); } - - return pattern; } } From 6facd1f9a8019bb3adf40bc4a3cec4990944ef56 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Thu, 16 Apr 2020 14:27:42 -0400 Subject: [PATCH 27/27] Address PR feedback, adding comments --- .../endpoint/scripts/resolver_generator.ts | 2 -- .../plugins/endpoint/server/index_pattern.ts | 25 +++++++++++++++++++ x-pack/plugins/endpoint/server/mocks.ts | 16 ++++++++++++ .../server/routes/alerts/details/handlers.ts | 4 --- .../server/routes/alerts/list/lib/index.ts | 2 -- .../server/routes/metadata/metadata.test.ts | 5 ++++ 6 files changed, 46 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/endpoint/scripts/resolver_generator.ts b/x-pack/plugins/endpoint/scripts/resolver_generator.ts index 0ecd756048d13..dd9e591f4b034 100644 --- a/x-pack/plugins/endpoint/scripts/resolver_generator.ts +++ b/x-pack/plugins/endpoint/scripts/resolver_generator.ts @@ -35,8 +35,6 @@ async function main() { eventIndex: { alias: 'ei', describe: 'index to store events in', - // This should be changed to events-endpoint-default-1 when this issue gets resolved: - // https://github.com/elastic/endpoint-app-team/issues/311 default: 'events-endpoint-1', type: 'string', }, diff --git a/x-pack/plugins/endpoint/server/index_pattern.ts b/x-pack/plugins/endpoint/server/index_pattern.ts index b35413598cc63..ea612bfd75441 100644 --- a/x-pack/plugins/endpoint/server/index_pattern.ts +++ b/x-pack/plugins/endpoint/server/index_pattern.ts @@ -13,6 +13,11 @@ export interface IndexPatternRetriever { getMetadataIndexPattern(ctx: RequestHandlerContext): Promise; } +/** + * This class is used to retrieve an index pattern. It should be used in the server side code whenever + * an index pattern is needed to query data within ES. The index pattern is constructed by the Ingest Manager + * based on the contents of the Endpoint Package in the Package Registry. + */ export class IngestIndexPatternRetriever implements IndexPatternRetriever { private static endpointPackageName = 'endpoint'; private static metadataDataset = 'metadata'; @@ -21,14 +26,34 @@ export class IngestIndexPatternRetriever implements IndexPatternRetriever { this.log = loggerFactory.get('index-pattern-retriever'); } + /** + * Retrieves the index pattern for querying events within elasticsearch. + * + * @param ctx a RequestHandlerContext from a route handler + * @returns a string representing the index pattern (e.g. `events-endpoint-*`) + */ async getEventIndexPattern(ctx: RequestHandlerContext) { return await this.getIndexPattern(ctx, EndpointAppConstants.EVENT_DATASET); } + /** + * Retrieves the index pattern for querying endpoint metadata within elasticsearch. + * + * @param ctx a RequestHandlerContext from a route handler + * @returns a string representing the index pattern (e.g. `metrics-endpoint-*`) + */ async getMetadataIndexPattern(ctx: RequestHandlerContext) { return await this.getIndexPattern(ctx, IngestIndexPatternRetriever.metadataDataset); } + /** + * Retrieves the index pattern for a specific dataset for querying endpoint data. + * + * @param ctx a RequestHandlerContext from a route handler + * @param datasetPath a string of the path being used for a dataset within the Endpoint Package + * (e.g. `events`, `metadata`) + * @returns a string representing the index pattern (e.g. `metrics-endpoint-*`) + */ async getIndexPattern(ctx: RequestHandlerContext, datasetPath: string) { try { const pattern = await this.service.getESIndexPattern( diff --git a/x-pack/plugins/endpoint/server/mocks.ts b/x-pack/plugins/endpoint/server/mocks.ts index 5c8aba0678577..903aa19cd8843 100644 --- a/x-pack/plugins/endpoint/server/mocks.ts +++ b/x-pack/plugins/endpoint/server/mocks.ts @@ -4,6 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ +/** + * Creates a mock IndexPatternRetriever for use in tests. + * + * @param indexPattern a string index pattern to return when any of the mock's public methods are called. + * @returns the same string passed in via `indexPattern` + */ export const createMockIndexPatternRetriever = (indexPattern: string) => { const mockGetFunc = jest.fn().mockResolvedValue(indexPattern); return { @@ -15,10 +21,20 @@ export const createMockIndexPatternRetriever = (indexPattern: string) => { export const MetadataIndexPattern = 'metrics-endpoint-*'; +/** + * Creates a mock IndexPatternRetriever for use in tests that returns `metrics-endpoint-*` + */ export const createMockMetadataIndexPatternRetriever = () => { return createMockIndexPatternRetriever(MetadataIndexPattern); }; +/** + * Creates a mock IndexPatternService for use in tests that need to interact with the Ingest Manager's + * ESIndexPatternService. + * + * @param indexPattern a string index pattern to return when called by a test + * @returns the same value as `indexPattern` parameter + */ export const createMockIndexPatternService = (indexPattern: string) => { return { esIndexPatternService: { diff --git a/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts b/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts index 8ed673d20bb9c..86e9f55da5697 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts @@ -22,8 +22,6 @@ export const alertDetailsHandlerWrapper = function( try { const alertId = req.params.id; const response = (await ctx.core.elasticsearch.dataClient.callAsCurrentUser('get', { - // Remove the hard coded reference to an alert index here - // https://github.com/elastic/endpoint-app-team/issues/311 index: EndpointAppConstants.ALERT_INDEX_NAME, id: alertId, })) as GetResponse; @@ -43,8 +41,6 @@ export const alertDetailsHandlerWrapper = function( return res.ok({ body: { - // base64 encode the index in the response - // https://github.com/elastic/endpoint-app-team/issues/311 id: response._id, ...response._source, state: { diff --git a/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts b/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts index a1b6e8eac7d04..95c8e4662cfce 100644 --- a/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts +++ b/x-pack/plugins/endpoint/server/routes/alerts/list/lib/index.ts @@ -108,8 +108,6 @@ export async function mapToAlertResultList( function mapHit(entry: AlertHits[0]): AlertData { return { - // base64 encode the alert index in the id field here - // https://github.com/elastic/endpoint-app-team/issues/311 id: entry._id, ...entry._source, prev: null, diff --git a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts index b38339b5f1475..9a7d3fb3188a6 100644 --- a/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/endpoint/server/routes/metadata/metadata.test.ts @@ -65,6 +65,11 @@ describe('test endpoint route', () => { client: savedObjectsClient, }, }, + /** + * Using unknown here because the object defined is not a full `RequestHandlerContext`. We don't + * need all of the fields required to run the tests, but the `routeHandler` function requires a + * `RequestHandlerContext`. + */ } as unknown) as RequestHandlerContext; }