From e67d5e702d36f9192f99dc1f9913bf74b8850e44 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 15 Jun 2020 17:58:55 -0700 Subject: [PATCH] Adjust agent config generation, change `datasources` to `inputs` --- .../common/services/config_to_yaml.ts | 5 +- .../datasource_to_agent_datasource.test.ts | 190 ------------------ .../datasource_to_agent_datasource.ts | 60 ------ .../datasources_to_agent_inputs.test.ts | 158 +++++++++++++++ .../services/datasources_to_agent_inputs.ts | 138 +++++++++++++ .../ingest_manager/common/services/index.ts | 2 +- .../common/types/models/agent_config.ts | 38 ++-- .../ingest_manager/services/index.ts | 2 +- .../server/services/agent_config.test.ts | 6 +- .../server/services/agent_config.ts | 6 +- .../ingest_manager/server/types/index.tsx | 2 +- 11 files changed, 323 insertions(+), 284 deletions(-) delete mode 100644 x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts delete mode 100644 x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts create mode 100644 x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts create mode 100644 x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts diff --git a/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts b/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts index 9dfd76b9ddd21..a3bef72e8db5a 100644 --- a/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts +++ b/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts @@ -11,10 +11,11 @@ const CONFIG_KEYS_ORDER = [ 'name', 'revision', 'type', - 'outputs', 'settings', - 'datasources', + 'outputs', + 'inputs', 'enabled', + 'use_output', 'package', 'input', ]; diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts deleted file mode 100644 index d319ba2beddf9..0000000000000 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Datasource, DatasourceInput } from '../types'; -import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource'; - -describe('Ingest Manager - storedDatasourceToAgentDatasource', () => { - const mockDatasource: Datasource = { - id: 'some-uuid', - name: 'mock-datasource', - description: '', - created_at: '', - created_by: '', - updated_at: '', - updated_by: '', - config_id: '', - enabled: true, - output_id: '', - namespace: 'default', - inputs: [], - revision: 1, - }; - - const mockInput: DatasourceInput = { - type: 'test-logs', - enabled: true, - vars: { - inputVar: { value: 'input-value' }, - inputVar2: { value: undefined }, - inputVar3: { - type: 'yaml', - value: 'testField: test', - }, - inputVar4: { value: '' }, - }, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - vars: { - fooVar: { value: 'foo-value' }, - fooVar2: { value: [1, 2] }, - }, - agent_stream: { - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - }, - { - id: 'test-logs-bar', - enabled: true, - dataset: 'bar', - vars: { - barVar: { value: 'bar-value' }, - barVar2: { value: [1, 2] }, - barVar3: { - type: 'yaml', - value: - '- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2', - }, - barVar4: { - type: 'yaml', - value: '', - }, - barVar5: { - type: 'yaml', - value: 'testField: test\n invalidSpacing: foo', - }, - }, - }, - ], - }; - - it('returns agent datasource config for datasource with no inputs', () => { - expect(storedDatasourceToAgentDatasource(mockDatasource)).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [], - }); - - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - package: { - name: 'mock-package', - title: 'Mock package', - version: '0.0.0', - }, - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - package: { - name: 'mock-package', - version: '0.0.0', - }, - inputs: [], - }); - }); - - it('returns agent datasource config with flattened input and package stream', () => { - expect(storedDatasourceToAgentDatasource({ ...mockDatasource, inputs: [mockInput] })).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [ - { - type: 'test-logs', - enabled: true, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - { - id: 'test-logs-bar', - enabled: true, - dataset: 'bar', - }, - ], - }, - ], - }); - }); - - it('returns agent datasource config without disabled streams', () => { - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - inputs: [ - { - ...mockInput, - streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }], - }, - ], - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [ - { - type: 'test-logs', - enabled: true, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - ], - }, - ], - }); - }); - - it('returns agent datasource config without disabled inputs', () => { - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - inputs: [{ ...mockInput, enabled: false }], - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [], - }); - }); -}); diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts deleted file mode 100644 index 2a8b687675bf9..0000000000000 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Datasource, FullAgentConfigDatasource } from '../types'; -import { DEFAULT_OUTPUT } from '../constants'; - -export const storedDatasourceToAgentDatasource = ( - datasource: Datasource -): FullAgentConfigDatasource => { - const { id, name, namespace, enabled, package: pkg, inputs } = datasource; - - const fullDatasource: FullAgentConfigDatasource = { - id: id || name, - name, - namespace, - enabled, - use_output: DEFAULT_OUTPUT.name, // TODO: hardcoded to default output for now - inputs: inputs - .filter((input) => input.enabled) - .map((input) => { - const fullInput = { - ...input, - ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => { - acc[key] = value; - return acc; - }, {} as { [k: string]: any }), - streams: input.streams - .filter((stream) => stream.enabled) - .map((stream) => { - const fullStream = { - ...stream, - ...stream.agent_stream, - ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { - acc[key] = value; - return acc; - }, {} as { [k: string]: any }), - }; - delete fullStream.agent_stream; - delete fullStream.vars; - delete fullStream.config; - return fullStream; - }), - }; - delete fullInput.vars; - delete fullInput.config; - return fullInput; - }), - }; - - if (pkg) { - fullDatasource.package = { - name: pkg.name, - version: pkg.version, - }; - } - - return fullDatasource; -}; diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts new file mode 100644 index 0000000000000..df94168ec88d0 --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Datasource, DatasourceInput } from '../types'; +import { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs'; + +describe('Ingest Manager - storedDatasourcesToAgentInputs', () => { + const mockDatasource: Datasource = { + id: 'some-uuid', + name: 'mock-datasource', + description: '', + created_at: '', + created_by: '', + updated_at: '', + updated_by: '', + config_id: '', + enabled: true, + output_id: '', + namespace: 'default', + inputs: [], + revision: 1, + }; + + const mockInput: DatasourceInput = { + type: 'test-logs', + enabled: true, + vars: { + inputVar: { value: 'input-value' }, + inputVar2: { value: undefined }, + inputVar3: { + type: 'yaml', + value: 'testField: test', + }, + inputVar4: { value: '' }, + }, + streams: [ + { + id: 'test-logs-foo', + enabled: true, + dataset: 'foo', + vars: { + fooVar: { value: 'foo-value' }, + fooVar2: { value: [1, 2] }, + }, + agent_stream: { + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + }, + { + id: 'test-logs-bar', + enabled: true, + dataset: 'bar', + vars: { + barVar: { value: 'bar-value' }, + barVar2: { value: [1, 2] }, + barVar3: { + type: 'yaml', + value: + '- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2', + }, + barVar4: { + type: 'yaml', + value: '', + }, + barVar5: { + type: 'yaml', + value: 'testField: test\n invalidSpacing: foo', + }, + }, + }, + ], + }; + + it('returns no inputs for datasource with no inputs, or only disabled inputs', () => { + expect(storedDatasourcesToAgentInputs([mockDatasource])).toEqual([]); + + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + package: { + name: 'mock-package', + title: 'Mock package', + version: '0.0.0', + }, + }, + ]) + ).toEqual([]); + + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + inputs: [{ ...mockInput, enabled: false }], + }, + ]) + ).toEqual([]); + }); + + it('returns agent inputs', () => { + expect(storedDatasourcesToAgentInputs([{ ...mockDatasource, inputs: [mockInput] }])).toEqual([ + { + id: 'some-uuid', + name: 'mock-datasource', + type: 'test-logs', + dataset: { namespace: 'default' }, + use_output: 'default', + streams: [ + { + id: 'test-logs-foo', + dataset: { name: 'foo' }, + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + { + id: 'test-logs-bar', + dataset: { name: 'bar' }, + }, + ], + }, + ]); + }); + + it('returns agent inputs without disabled streams', () => { + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + inputs: [ + { + ...mockInput, + streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }], + }, + ], + }, + ]) + ).toEqual([ + { + id: 'some-uuid', + name: 'mock-datasource', + type: 'test-logs', + dataset: { namespace: 'default' }, + use_output: 'default', + streams: [ + { + id: 'test-logs-foo', + dataset: { name: 'foo' }, + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + ], + }, + ]); + }); +}); diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts new file mode 100644 index 0000000000000..1c677722c3bcd --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Datasource, FullAgentConfigInput, FullAgentConfigInputStream } from '../types'; +import { DEFAULT_OUTPUT } from '../constants'; + +export const storedDatasourcesToAgentInputs = ( + datasources: Datasource[] +): FullAgentConfigInput[] => { + const fullInputs: FullAgentConfigInput[] = []; + + datasources.forEach((datasource) => { + if (!datasource.enabled || !datasource.inputs || !datasource.inputs.length) { + return; + } + datasource.inputs.forEach((input) => { + if (!input.enabled) { + return; + } + + const fullInput: FullAgentConfigInput = { + id: datasource.id || datasource.name, + name: datasource.name, + type: input.type, + dataset: { namespace: datasource.namespace || 'default' }, + use_output: DEFAULT_OUTPUT.name, + ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => { + acc[key] = value; + return acc; + }, {} as { [k: string]: any }), + streams: input.streams + .filter((stream) => stream.enabled) + .map((stream) => { + const fullStream: FullAgentConfigInputStream = { + id: stream.id, + dataset: { name: stream.dataset }, + ...stream.agent_stream, + ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { + acc[key] = value; + return acc; + }, {} as { [k: string]: any }), + }; + if (stream.processors) { + fullStream.processors = stream.processors; + } + return fullStream; + }), + }; + + if (datasource.package) { + fullInput.package = { + name: datasource.package.name, + version: datasource.package.version, + }; + } + + fullInputs.push(fullInput); + }); + }); + + return fullInputs; +}; + +// export const storedDatasourcesToAgentInputs = (datasource: Datasource): FullAgentConfigInput => { +// const { id, name, namespace, enabled, package: pkg, inputs } = datasource; + +// const fullStreams: FullAgentConfigInputStream[] = []; + +// inputs.forEach((input) => { +// if (!input.enabled || !input.streams || !input.streams.length) { +// return; +// } +// const streams: FullAgentConfigInputStream[] = input.streams +// .filter((stream) => stream.enabled) +// .map((stream) => { +// const fullStream: FullAgentConfigInputStream = { +// id: stream.id, +// processors: stream.processors, +// 'dataset.type': stream.dataset, +// ...stream.agent_stream, +// ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { +// acc[key] = value; +// return acc; +// }, {} as { [k: string]: any }), +// }; +// return fullStream; +// }); +// fullStreams.concat(streams); +// }); + +// const fullInput: FullAgentConfigInput = { +// id: id || name, +// name, +// 'dataset.namespace': namespace, +// use_output: DEFAULT_OUTPUT.name, // TODO: hardcoded to default output for now +// streams: inputs +// .filter((input) => input.enabled) +// .map((input) => { +// const fullInput = { +// ...input, +// ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => { +// acc[key] = value; +// return acc; +// }, {} as { [k: string]: any }), +// streams: input.streams +// .filter((stream) => stream.enabled) +// .map((stream) => { +// const fullStream = { +// ...stream, +// ...stream.agent_stream, +// ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { +// acc[key] = value; +// return acc; +// }, {} as { [k: string]: any }), +// }; +// delete fullStream.agent_stream; +// delete fullStream.vars; +// delete fullStream.config; +// return fullStream; +// }), +// }; +// delete fullInput.vars; +// delete fullInput.config; +// return fullInput; +// }), +// }; + +// if (pkg) { +// fullInput.package = { +// name: pkg.name, +// version: pkg.version, +// }; +// } + +// return fullInput; +// }; diff --git a/x-pack/plugins/ingest_manager/common/services/index.ts b/x-pack/plugins/ingest_manager/common/services/index.ts index c595c9a52f66f..e53d97972fa2f 100644 --- a/x-pack/plugins/ingest_manager/common/services/index.ts +++ b/x-pack/plugins/ingest_manager/common/services/index.ts @@ -7,7 +7,7 @@ import * as AgentStatusKueryHelper from './agent_status'; export * from './routes'; export { packageToConfigDatasourceInputs, packageToConfigDatasource } from './package_to_config'; -export { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource'; +export { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs'; export { configToYaml } from './config_to_yaml'; export { AgentStatusKueryHelper }; export { decodeCloudId } from './decode_cloud_id'; diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts index 7547f56237eec..36b3176ffa415 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts @@ -3,12 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { - Datasource, - DatasourcePackage, - DatasourceInput, - DatasourceInputStream, -} from './datasource'; +import { Datasource, DatasourcePackage, DatasourceInputStream } from './datasource'; import { Output } from './output'; export enum AgentConfigStatus { @@ -35,23 +30,22 @@ export interface AgentConfig extends NewAgentConfig { export type AgentConfigSOAttributes = Omit; -export type FullAgentConfigDatasource = Pick< - Datasource, - 'id' | 'name' | 'namespace' | 'enabled' -> & { - package?: Pick; - use_output: string; - inputs: Array< - Omit & { - streams: Array< - Omit & { - [key: string]: any; - } - >; - } - >; +export type FullAgentConfigInputStream = Pick & { + dataset: { name: string }; + [key: string]: any; }; +export interface FullAgentConfigInput { + id: string; + name: string; + type: string; + dataset: { namespace: string }; + use_output: string; + package?: Pick; + streams: FullAgentConfigInputStream[]; + [key: string]: any; +} + export interface FullAgentConfig { id: string; outputs: { @@ -59,7 +53,7 @@ export interface FullAgentConfig { [key: string]: any; }; }; - datasources: FullAgentConfigDatasource[]; + inputs: FullAgentConfigInput[]; revision?: number; settings?: { monitoring: { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts index 6dbc8d67caaee..ece7aef2c247f 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts @@ -19,7 +19,7 @@ export { settingsRoutesService, appRoutesService, packageToConfigDatasourceInputs, - storedDatasourceToAgentDatasource, + storedDatasourcesToAgentInputs, configToYaml, AgentStatusKueryHelper, } from '../../../../common'; diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts index 17758f6e3d7f1..c46e648ad088a 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts @@ -59,7 +59,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { @@ -88,7 +88,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { @@ -118,7 +118,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts index 4a877ef7de13b..5ed75c6b52cdf 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts @@ -20,7 +20,7 @@ import { AgentConfigStatus, ListWithKuery, } from '../types'; -import { DeleteAgentConfigResponse, storedDatasourceToAgentDatasource } from '../../common'; +import { DeleteAgentConfigResponse, storedDatasourcesToAgentInputs } from '../../common'; import { listAgents } from './agents'; import { datasourceService } from './datasource'; import { outputService } from './output'; @@ -374,9 +374,7 @@ class AgentConfigService { {} as FullAgentConfig['outputs'] ), }, - datasources: (config.datasources as Datasource[]) - .filter((datasource) => datasource.enabled) - .map((ds) => storedDatasourceToAgentDatasource(ds)), + inputs: storedDatasourcesToAgentInputs(config.datasources as Datasource[]), revision: config.revision, ...(config.monitoring_enabled && config.monitoring_enabled.length > 0 ? { diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 2218d967fa8aa..2b543490ca8da 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -20,7 +20,7 @@ export { Datasource, NewDatasource, DatasourceSOAttributes, - FullAgentConfigDatasource, + FullAgentConfigInput, FullAgentConfig, AgentConfig, AgentConfigSOAttributes,