diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts index 9201cdcb6bbac..7b4e4adc4e4fc 100644 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts +++ b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { NewDatasource } from '../types'; +import { NewDatasource, DatasourceInput } from '../types'; import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource'; describe('Ingest Manager - storedDatasourceToAgentDatasource', () => { @@ -17,7 +17,7 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => { inputs: [], }; - const mockInput = { + const mockInput: DatasourceInput = { type: 'test-logs', enabled: true, streams: [ @@ -25,13 +25,29 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => { id: 'test-logs-foo', enabled: true, dataset: 'foo', - config: { fooVar: 'foo-value', fooVar2: [1, 2] }, + config: { fooVar: { value: 'foo-value' }, fooVar2: { value: [1, 2] } }, }, { id: 'test-logs-bar', enabled: false, dataset: 'bar', - config: { barVar: 'bar-value', barVar2: [1, 2] }, + config: { + barVar: { value: 'bar-value' }, + barVar2: { value: [1, 2] }, + barVar3: { + type: 'yaml', + value: + '- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2', + }, + barVar4: { + type: 'yaml', + value: '', + }, + barVar5: { + type: 'yaml', + value: 'testField: test\n invalidSpacing: foo', + }, + }, }, ], }; @@ -91,6 +107,16 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => { dataset: 'bar', barVar: 'bar-value', barVar2: [1, 2], + barVar3: [ + { + namespace: 'mockNamespace', + anotherProp: 'test', + }, + { + namespace: 'mockNamespace2', + anotherProp: 'test2', + }, + ], }, ], }, diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts index 57627fa60fe43..ea048b84afef3 100644 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts +++ b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { safeLoad } from 'js-yaml'; import { Datasource, NewDatasource, FullAgentConfigDatasource } from '../types'; import { DEFAULT_OUTPUT } from '../constants'; @@ -23,12 +24,26 @@ export const storedDatasourceToAgentDatasource = ( if (stream.config) { const fullStream = { ...stream, - ...Object.entries(stream.config).reduce((acc, [configName, configValue]) => { - if (configValue !== undefined) { - acc[configName] = configValue; - } - return acc; - }, {} as { [key: string]: any }), + ...Object.entries(stream.config).reduce( + (acc, [configName, { type: configType, value: configValue }]) => { + if (configValue !== undefined) { + if (configType === 'yaml') { + try { + const yamlValue = safeLoad(configValue); + if (yamlValue) { + acc[configName] = yamlValue; + } + } catch (e) { + // Silently swallow parsing error + } + } else { + acc[configName] = configValue; + } + } + return acc; + }, + {} as { [key: string]: any } + ), }; delete fullStream.config; return fullStream; diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts index d312e7aa35cc0..e54e59dd24df3 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts @@ -108,8 +108,14 @@ describe('Ingest Manager - packageToConfig', () => { { type: 'bar', streams: [ - { dataset: 'bar', vars: [{ default: 'bar-var-value', name: 'var-name' }] }, - { dataset: 'bar2', vars: [{ default: 'bar2-var-value', name: 'var-name' }] }, + { + dataset: 'bar', + vars: [{ default: 'bar-var-value', name: 'var-name', type: 'text' }], + }, + { + dataset: 'bar2', + vars: [{ default: 'bar2-var-value', name: 'var-name', type: 'yaml' }], + }, ], }, ], @@ -125,7 +131,7 @@ describe('Ingest Manager - packageToConfig', () => { id: 'foo-foo', enabled: true, dataset: 'foo', - config: { 'var-name': 'foo-var-value' }, + config: { 'var-name': { value: 'foo-var-value' } }, }, ], }, @@ -137,13 +143,13 @@ describe('Ingest Manager - packageToConfig', () => { id: 'bar-bar', enabled: true, dataset: 'bar', - config: { 'var-name': 'bar-var-value' }, + config: { 'var-name': { type: 'text', value: 'bar-var-value' } }, }, { id: 'bar-bar2', enabled: true, dataset: 'bar2', - config: { 'var-name': 'bar2-var-value' }, + config: { 'var-name': { type: 'yaml', value: 'bar2-var-value' } }, }, ], }, @@ -204,10 +210,10 @@ describe('Ingest Manager - packageToConfig', () => { enabled: true, dataset: 'foo', config: { - 'var-name': 'foo-var-value', - 'foo-input-var-name': 'foo-input-var-value', - 'foo-input2-var-name': 'foo-input2-var-value', - 'foo-input3-var-name': undefined, + 'var-name': { value: 'foo-var-value' }, + 'foo-input-var-name': { value: 'foo-input-var-value' }, + 'foo-input2-var-name': { value: 'foo-input2-var-value' }, + 'foo-input3-var-name': { value: undefined }, }, }, ], @@ -221,9 +227,9 @@ describe('Ingest Manager - packageToConfig', () => { enabled: true, dataset: 'bar', config: { - 'var-name': 'bar-var-value', - 'bar-input-var-name': ['value1', 'value2'], - 'bar-input2-var-name': 123456, + 'var-name': { value: 'bar-var-value' }, + 'bar-input-var-name': { value: ['value1', 'value2'] }, + 'bar-input2-var-name': { value: 123456 }, }, }, { @@ -231,9 +237,9 @@ describe('Ingest Manager - packageToConfig', () => { enabled: true, dataset: 'bar2', config: { - 'var-name': 'bar2-var-value', - 'bar-input-var-name': ['value1', 'value2'], - 'bar-input2-var-name': 123456, + 'var-name': { value: 'bar2-var-value' }, + 'bar-input-var-name': { value: ['value1', 'value2'] }, + 'bar-input2-var-name': { value: 123456 }, }, }, ], @@ -247,7 +253,7 @@ describe('Ingest Manager - packageToConfig', () => { enabled: false, dataset: 'disabled', config: { - 'var-name': [], + 'var-name': { value: [] }, }, }, { diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.ts index 9785edbff1112..6de75a004303e 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_config.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_config.ts @@ -41,9 +41,9 @@ export const packageToConfigDatasourceInputs = (packageInfo: PackageInfo): Datas streamVar: RegistryVarsEntry ): DatasourceInputStream['config'] => { if (!streamVar.default && streamVar.multi) { - configObject![streamVar.name] = []; + configObject![streamVar.name] = { type: streamVar.type, value: [] }; } else { - configObject![streamVar.name] = streamVar.default; + configObject![streamVar.name] = { type: streamVar.type, value: streamVar.default }; } return configObject; }; diff --git a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts index 3503bbdcd40e3..3ad7a15d0c739 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts @@ -15,7 +15,13 @@ export interface DatasourceInputStream { enabled: boolean; dataset: string; processors?: string[]; - config?: Record; + config?: Record< + string, + { + type?: string; + value: any; + } + >; } export interface DatasourceInput { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_config.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_config.tsx index 69d2194638441..1128f25818d7c 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_config.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_config.tsx @@ -63,8 +63,8 @@ export const DatasourceInputConfig: React.FunctionComponent<{ {requiredVars.map(varDef => { - const varName = varDef.name; - const value = datasourceInput.streams[0].config![varName]; + const { name: varName, type: varType } = varDef; + const value = datasourceInput.streams[0].config![varName].value; return ( {isShowingAdvanced ? advancedVars.map(varDef => { - const varName = varDef.name; - const value = datasourceInput.streams[0].config![varName]; + const { name: varName, type: varType } = varDef; + const value = datasourceInput.streams[0].config![varName].value; return ( {requiredVars.map(varDef => { - const varName = varDef.name; - const value = datasourceInputStream.config![varName]; + const { name: varName, type: varType } = varDef; + const value = datasourceInputStream.config![varName].value; return ( {isShowingAdvanced ? advancedVars.map(varDef => { - const varName = varDef.name; - const value = datasourceInputStream.config![varName]; + const { name: varName, type: varType } = varDef; + const value = datasourceInputStream.config![varName].value; return ( void; }> = ({ varDef, value, onChange }) => { + const renderField = () => { + if (varDef.multi) { + return ( + ({ label: val }))} + onCreateOption={(newVal: any) => { + onChange([...value, newVal]); + }} + onChange={(newVals: any[]) => { + onChange(newVals.map(val => val.label)); + }} + /> + ); + } + if (varDef.type === 'yaml') { + return ( + onChange(newVal)} + /> + ); + } + return ( + onChange(e.target.value)} + /> + ); + }; + return ( } > - {varDef.multi ? ( - ({ label: val }))} - onCreateOption={(newVal: any) => { - onChange([...value, newVal]); - }} - onChange={(newVals: any[]) => { - onChange(newVals.map(val => val.label)); - }} - /> - ) : ( - onChange(e.target.value)} /> - )} + {renderField()} ); }; diff --git a/x-pack/plugins/ingest_manager/server/types/models/datasource.ts b/x-pack/plugins/ingest_manager/server/types/models/datasource.ts index 94d0a1cc1aabf..51687016f6aad 100644 --- a/x-pack/plugins/ingest_manager/server/types/models/datasource.ts +++ b/x-pack/plugins/ingest_manager/server/types/models/datasource.ts @@ -31,7 +31,13 @@ const DatasourceBaseSchema = { enabled: schema.boolean(), dataset: schema.string(), processors: schema.maybe(schema.arrayOf(schema.string())), - config: schema.recordOf(schema.string(), schema.any()), + config: schema.recordOf( + schema.string(), + schema.object({ + type: schema.maybe(schema.string()), + value: schema.any(), + }) + ), }) ), })