diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts index df94168ec88d0..f587c4bc3bb9e 100644 --- a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts +++ b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts @@ -39,7 +39,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => { { id: 'test-logs-foo', enabled: true, - dataset: 'foo', + dataset: { name: 'foo' }, vars: { fooVar: { value: 'foo-value' }, fooVar2: { value: [1, 2] }, @@ -52,7 +52,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => { { id: 'test-logs-bar', enabled: true, - dataset: 'bar', + dataset: { name: 'bar' }, vars: { barVar: { value: 'bar-value' }, barVar2: { value: [1, 2] }, diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts index d5a752e817b4f..0731288f0f06a 100644 --- a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts +++ b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts @@ -24,7 +24,10 @@ export const storedDatasourcesToAgentInputs = ( id: datasource.id || datasource.name, name: datasource.name, type: input.type, - dataset: { namespace: datasource.namespace || 'default' }, + dataset: { + namespace: datasource.namespace || 'default', + ...(input.dataset?.type ? { type: input.dataset.type } : {}), + }, use_output: DEFAULT_OUTPUT.name, ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => { acc[key] = value; @@ -35,7 +38,7 @@ export const storedDatasourcesToAgentInputs = ( .map((stream) => { const fullStream: FullAgentConfigInputStream = { id: stream.id, - dataset: { name: stream.dataset }, + dataset: stream.dataset, ...stream.agent_stream, ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { acc[key] = value; diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts index 2a991c100d07e..0d0fcb637eca5 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts @@ -82,15 +82,17 @@ describe('Ingest Manager - packageToConfig', () => { ).toEqual([ { type: 'foo', + dataset: { type: 'logs' }, enabled: true, - streams: [{ id: 'foo-foo', enabled: true, dataset: 'foo' }], + streams: [{ id: 'foo-foo', enabled: true, dataset: { name: 'foo' } }], }, { type: 'bar', + dataset: { type: 'logs' }, enabled: true, streams: [ - { id: 'bar-bar', enabled: true, dataset: 'bar' }, - { id: 'bar-bar2', enabled: true, dataset: 'bar2' }, + { id: 'bar-bar', enabled: true, dataset: { name: 'bar' } }, + { id: 'bar-bar2', enabled: true, dataset: { name: 'bar2' } }, ], }, ]); @@ -136,30 +138,32 @@ describe('Ingest Manager - packageToConfig', () => { ).toEqual([ { type: 'foo', + dataset: { type: 'logs' }, enabled: true, streams: [ { id: 'foo-foo', enabled: true, - dataset: 'foo', + dataset: { name: 'foo' }, vars: { 'var-name': { value: 'foo-var-value' } }, }, ], }, { type: 'bar', + dataset: { type: 'logs' }, enabled: true, streams: [ { id: 'bar-bar', enabled: true, - dataset: 'bar', + dataset: { name: 'bar' }, vars: { 'var-name': { type: 'text', value: 'bar-var-value' } }, }, { id: 'bar-bar2', enabled: true, - dataset: 'bar2', + dataset: { name: 'bar2' }, vars: { 'var-name': { type: 'yaml', value: 'bar2-var-value' } }, }, ], @@ -247,6 +251,7 @@ describe('Ingest Manager - packageToConfig', () => { ).toEqual([ { type: 'foo', + dataset: { type: 'logs' }, enabled: true, vars: { 'foo-input-var-name': { value: 'foo-input-var-value' }, @@ -257,7 +262,7 @@ describe('Ingest Manager - packageToConfig', () => { { id: 'foo-foo', enabled: true, - dataset: 'foo', + dataset: { name: 'foo' }, vars: { 'var-name': { value: 'foo-var-value' }, }, @@ -266,6 +271,7 @@ describe('Ingest Manager - packageToConfig', () => { }, { type: 'bar', + dataset: { type: 'logs' }, enabled: true, vars: { 'bar-input-var-name': { value: ['value1', 'value2'] }, @@ -275,7 +281,7 @@ describe('Ingest Manager - packageToConfig', () => { { id: 'bar-bar', enabled: true, - dataset: 'bar', + dataset: { name: 'bar' }, vars: { 'var-name': { value: 'bar-var-value' }, }, @@ -283,7 +289,7 @@ describe('Ingest Manager - packageToConfig', () => { { id: 'bar-bar2', enabled: true, - dataset: 'bar2', + dataset: { name: 'bar2' }, vars: { 'var-name': { value: 'bar2-var-value' }, }, @@ -292,12 +298,13 @@ describe('Ingest Manager - packageToConfig', () => { }, { type: 'with-disabled-streams', + dataset: { type: 'logs' }, enabled: false, streams: [ { id: 'with-disabled-streams-disabled', enabled: false, - dataset: 'disabled', + dataset: { name: 'disabled' }, vars: { 'var-name': { value: [] }, }, @@ -305,7 +312,7 @@ describe('Ingest Manager - packageToConfig', () => { { id: 'with-disabled-streams-disabled2', enabled: false, - dataset: 'disabled2', + dataset: { name: 'disabled2' }, }, ], }, diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.ts index 8337e5c0e46e5..d204139e21776 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_config.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_config.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { pluck, uniq } from 'lodash'; import { PackageInfo, RegistryConfigTemplate, @@ -16,11 +17,11 @@ import { NewDatasource, } from '../types'; -const findStreamsForInputType = ( +const getStreamsForInputType = ( inputType: string, packageInfo: PackageInfo -): Array => { - const streams: Array = []; +): Array => { + const streams: Array = []; (packageInfo.datasets || []).forEach((dataset) => { (dataset.streams || []).forEach((stream) => { @@ -28,6 +29,7 @@ const findStreamsForInputType = ( streams.push({ ...stream, dataset: { + type: dataset.type, name: dataset.name, }, }); @@ -38,6 +40,11 @@ const findStreamsForInputType = ( return streams; }; +const getDatasetTypeForInputType = (inputType: string, packageInfo: PackageInfo): string => { + const datasetTypes = uniq(pluck(getStreamsForInputType(inputType, packageInfo), 'dataset.type')); + return datasetTypes[0]; +}; + /* * This service creates a datasource inputs definition from defaults provided in package info */ @@ -70,14 +77,14 @@ export const packageToConfigDatasourceInputs = (packageInfo: PackageInfo): Datas }; // Map each package input stream into datasource input stream - const streams: DatasourceInputStream[] = findStreamsForInputType( + const streams: DatasourceInputStream[] = getStreamsForInputType( packageInput.type, packageInfo ).map((packageStream) => { const stream: DatasourceInputStream = { id: `${packageInput.type}-${packageStream.dataset.name}`, enabled: packageStream.enabled === false ? false : true, - dataset: packageStream.dataset.name, + dataset: { name: packageStream.dataset.name }, }; if (packageStream.vars && packageStream.vars.length) { stream.vars = packageStream.vars.reduce(varsReducer, {}); @@ -91,6 +98,12 @@ export const packageToConfigDatasourceInputs = (packageInfo: PackageInfo): Datas streams, }; + const datasetType = getDatasetTypeForInputType(packageInput.type, packageInfo); + + if (datasetType) { + input.dataset = { type: datasetType }; + } + if (packageInput.vars && packageInput.vars.length) { input.vars = packageInput.vars.reduce(varsReducer, {}); } diff --git a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts index aa92b90a6caec..f927dc34a5d65 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts @@ -20,7 +20,9 @@ export type DatasourceConfigRecord = Record export interface NewDatasourceInputStream { id: string; enabled: boolean; - dataset: string; + dataset: { + name: string; + }; processors?: string[]; config?: DatasourceConfigRecord; vars?: DatasourceConfigRecord; @@ -32,6 +34,9 @@ export interface DatasourceInputStream extends NewDatasourceInputStream { export interface NewDatasourceInput { type: string; + dataset?: { + type: string; + }; enabled: boolean; processors?: string[]; config?: DatasourceConfigRecord; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_panel.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_panel.tsx index 78ae314e209b7..6f6fa5aaa7f3e 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_panel.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_panel.tsx @@ -177,7 +177,7 @@ export const DatasourceInputPanel: React.FunctionComponent<{ {packageInputStreams.map((packageInputStream) => { const datasourceInputStream = datasourceInput.streams.find( - (stream) => stream.dataset === packageInputStream.dataset.name + (stream) => stream.dataset.name === packageInputStream.dataset.name ); return datasourceInputStream ? ( @@ -186,7 +186,7 @@ export const DatasourceInputPanel: React.FunctionComponent<{ datasourceInputStream={datasourceInputStream} updateDatasourceInputStream={(updatedStream: Partial) => { const indexOfUpdatedStream = datasourceInput.streams.findIndex( - (stream) => stream.dataset === packageInputStream.dataset.name + (stream) => stream.dataset.name === packageInputStream.dataset.name ); const newStreams = [...datasourceInput.streams]; newStreams[indexOfUpdatedStream] = { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.test.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.test.ts index 70e7b3e53601b..b5e3677ba8f67 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.test.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.test.ts @@ -158,7 +158,7 @@ describe('Ingest Manager - validateDatasource()', () => { streams: [ { id: 'foo-foo', - dataset: 'foo', + dataset: { name: 'foo' }, enabled: true, vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } }, }, @@ -174,13 +174,13 @@ describe('Ingest Manager - validateDatasource()', () => { streams: [ { id: 'bar-bar', - dataset: 'bar', + dataset: { name: 'bar' }, enabled: true, vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } }, }, { id: 'bar-bar2', - dataset: 'bar2', + dataset: { name: 'bar2' }, enabled: true, vars: { 'var-name': { value: undefined, type: 'text' } }, }, @@ -197,13 +197,13 @@ describe('Ingest Manager - validateDatasource()', () => { streams: [ { id: 'with-disabled-streams-disabled', - dataset: 'disabled', + dataset: { name: 'disabled' }, enabled: false, vars: { 'var-name': { value: undefined, type: 'text' } }, }, { id: 'with-disabled-streams-disabled-without-vars', - dataset: 'disabled2', + dataset: { name: 'disabled2' }, enabled: false, }, ], @@ -217,7 +217,7 @@ describe('Ingest Manager - validateDatasource()', () => { streams: [ { id: 'with-no-stream-vars-bar', - dataset: 'bar', + dataset: { name: 'bar' }, enabled: true, }, ], @@ -240,7 +240,7 @@ describe('Ingest Manager - validateDatasource()', () => { streams: [ { id: 'foo-foo', - dataset: 'foo', + dataset: { name: 'foo' }, enabled: true, vars: { 'var-name': { value: 'invalidyaml: test\n foo bar:', type: 'yaml' } }, }, @@ -256,13 +256,13 @@ describe('Ingest Manager - validateDatasource()', () => { streams: [ { id: 'bar-bar', - dataset: 'bar', + dataset: { name: 'bar' }, enabled: true, vars: { 'var-name': { value: ' \n\n', type: 'yaml' } }, }, { id: 'bar-bar2', - dataset: 'bar2', + dataset: { name: 'bar2' }, enabled: true, vars: { 'var-name': { value: undefined, type: 'text' } }, }, @@ -279,7 +279,7 @@ describe('Ingest Manager - validateDatasource()', () => { streams: [ { id: 'with-disabled-streams-disabled', - dataset: 'disabled', + dataset: { name: 'disabled' }, enabled: false, vars: { 'var-name': { @@ -290,7 +290,7 @@ describe('Ingest Manager - validateDatasource()', () => { }, { id: 'with-disabled-streams-disabled-without-vars', - dataset: 'disabled2', + dataset: { name: 'disabled2' }, enabled: false, }, ], @@ -304,7 +304,7 @@ describe('Ingest Manager - validateDatasource()', () => { streams: [ { id: 'with-no-stream-vars-bar', - dataset: 'bar', + dataset: { name: 'bar' }, enabled: true, }, ], diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.ts index bdd6c08a8ef57..30dca4a5fbf81 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.ts @@ -124,7 +124,7 @@ export const validateDatasource = ( if (stream.vars) { const streamVarsByName = ( ( - registryStreamsByDataset[stream.dataset].find( + registryStreamsByDataset[stream.dataset.name].find( (registryStream) => registryStream.input === input.type ) || {} ).vars || [] diff --git a/x-pack/plugins/ingest_manager/server/saved_objects/index.ts b/x-pack/plugins/ingest_manager/server/saved_objects/index.ts index 703ddb521c831..0c3a5de654d97 100644 --- a/x-pack/plugins/ingest_manager/server/saved_objects/index.ts +++ b/x-pack/plugins/ingest_manager/server/saved_objects/index.ts @@ -203,6 +203,11 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = { type: 'nested', properties: { type: { type: 'keyword' }, + dataset: { + properties: { + type: { type: 'keyword' }, + }, + }, enabled: { type: 'boolean' }, processors: { type: 'keyword' }, config: { type: 'flattened' }, @@ -212,7 +217,11 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = { properties: { id: { type: 'keyword' }, enabled: { type: 'boolean' }, - dataset: { type: 'keyword' }, + dataset: { + properties: { + name: { type: 'keyword' }, + }, + }, processors: { type: 'keyword' }, config: { type: 'flattened' }, agent_stream: { type: 'flattened' }, diff --git a/x-pack/plugins/ingest_manager/server/saved_objects/migrations/datasources_v790.ts b/x-pack/plugins/ingest_manager/server/saved_objects/migrations/datasources_v790.ts index 0d1fb6f21a1ae..e90fc3a6252d7 100644 --- a/x-pack/plugins/ingest_manager/server/saved_objects/migrations/datasources_v790.ts +++ b/x-pack/plugins/ingest_manager/server/saved_objects/migrations/datasources_v790.ts @@ -6,12 +6,22 @@ import { SavedObjectMigrationFn } from 'kibana/server'; import { cloneDeep } from 'lodash'; -import { Datasource } from '../../types/models'; +import { Datasource, DatasourceInput, DatasourceInputStream } from '../../types'; type Pre790Datasource = Exclude< Datasource, 'created_at' | 'created_by' | 'updated_at' | 'updated_by' ->; +> & { + inputs: Array< + Exclude & { + streams: Array< + Exclude & { + dataset: string; + } + >; + } + >; +}; export const migrateDatasourcesToV790: SavedObjectMigrationFn = ( doc @@ -23,6 +33,11 @@ export const migrateDatasourcesToV790: SavedObjectMigrationFn { + input.streams.forEach((stream) => { + stream.dataset = { name: (stream.dataset as unknown) as string }; + }); + }); return updatedDatasource; }; diff --git a/x-pack/plugins/ingest_manager/server/services/datasource.test.ts b/x-pack/plugins/ingest_manager/server/services/datasource.test.ts index 5d8982834284f..83e209ba07ea1 100644 --- a/x-pack/plugins/ingest_manager/server/services/datasource.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/datasource.test.ts @@ -38,7 +38,7 @@ describe('Datasource service', () => { ({ datasets: [ { - name: 'dataset1', + name: 'package.dataset1', streams: [{ input: 'log', template_path: 'some_template_path.yml' }], }, ], @@ -55,7 +55,7 @@ describe('Datasource service', () => { streams: [ { id: 'dataset01', - dataset: 'dataset1', + dataset: { name: 'package.dataset1' }, enabled: true, vars: { paths: { @@ -75,7 +75,7 @@ describe('Datasource service', () => { streams: [ { id: 'dataset01', - dataset: 'dataset1', + dataset: { name: 'package.dataset1' }, enabled: true, vars: { paths: { @@ -98,7 +98,7 @@ describe('Datasource service', () => { ({ datasets: [ { - name: 'dataset1', + name: 'package.dataset1', streams: [{ input: 'log', template_path: 'some_template_path.yml' }], }, ], @@ -120,7 +120,7 @@ describe('Datasource service', () => { streams: [ { id: 'dataset01', - dataset: 'dataset1', + dataset: { name: 'package.dataset1' }, enabled: true, }, ], @@ -140,7 +140,7 @@ describe('Datasource service', () => { streams: [ { id: 'dataset01', - dataset: 'dataset1', + dataset: { name: 'package.dataset1' }, enabled: true, agent_stream: { metricset: ['dataset1'], diff --git a/x-pack/plugins/ingest_manager/server/services/datasource.ts b/x-pack/plugins/ingest_manager/server/services/datasource.ts index 613e501d33de9..bfc57e282ca2f 100644 --- a/x-pack/plugins/ingest_manager/server/services/datasource.ts +++ b/x-pack/plugins/ingest_manager/server/services/datasource.ts @@ -275,13 +275,15 @@ async function _assignPackageStreamToStream( if (!stream.enabled) { return { ...stream, agent_stream: undefined }; } - const datasetPath = getDataset(stream.dataset); + const datasetPath = getDataset(stream.dataset.name); const packageDatasets = pkgInfo.datasets; if (!packageDatasets) { throw new Error('Stream template not found, no datasets'); } - const packageDataset = packageDatasets.find((pkgDataset) => pkgDataset.name === stream.dataset); + const packageDataset = packageDatasets.find( + (pkgDataset) => pkgDataset.name === stream.dataset.name + ); if (!packageDataset) { throw new Error(`Stream template not found, unable to find dataset ${datasetPath}`); } diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 2b543490ca8da..60c6aff6af458 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -18,6 +18,8 @@ export { AgentAction, AgentActionSOAttributes, Datasource, + DatasourceInput, + DatasourceInputStream, NewDatasource, DatasourceSOAttributes, FullAgentConfigInput, diff --git a/x-pack/plugins/ingest_manager/server/types/models/datasource.ts b/x-pack/plugins/ingest_manager/server/types/models/datasource.ts index 3bca6d20d96a2..a5dba6c97e0cf 100644 --- a/x-pack/plugins/ingest_manager/server/types/models/datasource.ts +++ b/x-pack/plugins/ingest_manager/server/types/models/datasource.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ import { schema } from '@kbn/config-schema'; -export { Datasource, NewDatasource } from '../../../common'; const ConfigRecordSchema = schema.recordOf( schema.string(), @@ -31,6 +30,7 @@ const DatasourceBaseSchema = { inputs: schema.arrayOf( schema.object({ type: schema.string(), + dataset: schema.maybe(schema.object({ type: schema.string() })), enabled: schema.boolean(), processors: schema.maybe(schema.arrayOf(schema.string())), vars: schema.maybe(ConfigRecordSchema), @@ -47,7 +47,7 @@ const DatasourceBaseSchema = { schema.object({ id: schema.string(), enabled: schema.boolean(), - dataset: schema.string(), + dataset: schema.object({ name: schema.string() }), processors: schema.maybe(schema.arrayOf(schema.string())), vars: schema.maybe(ConfigRecordSchema), config: schema.maybe( diff --git a/x-pack/plugins/ingest_manager/server/types/models/output.ts b/x-pack/plugins/ingest_manager/server/types/models/output.ts index 36b945db2cbce..22a101ecd94b8 100644 --- a/x-pack/plugins/ingest_manager/server/types/models/output.ts +++ b/x-pack/plugins/ingest_manager/server/types/models/output.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ import { schema } from '@kbn/config-schema'; -export { Output, NewOutput } from '../../../common'; export enum OutputType { Elasticsearch = 'elasticsearch',