diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts index 25180244b0214..cacf84381dd88 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts @@ -93,7 +93,7 @@ test('tests processing text field with multi fields', () => { const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(textWithMultiFieldsMapping)); + expect(mappings).toEqual(textWithMultiFieldsMapping); }); test('tests processing keyword field with multi fields', () => { @@ -127,7 +127,7 @@ test('tests processing keyword field with multi fields', () => { const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithMultiFieldsMapping)); + expect(mappings).toEqual(keywordWithMultiFieldsMapping); }); test('tests processing keyword field with multi fields with analyzed text field', () => { @@ -159,7 +159,7 @@ test('tests processing keyword field with multi fields with analyzed text field' const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithAnalyzedMultiFieldsMapping)); + expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); }); test('tests processing object field with no other attributes', () => { @@ -177,7 +177,7 @@ test('tests processing object field with no other attributes', () => { const fields: Field[] = safeLoad(objectFieldLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldMapping)); + expect(mappings).toEqual(objectFieldMapping); }); test('tests processing object field with enabled set to false', () => { @@ -197,7 +197,7 @@ test('tests processing object field with enabled set to false', () => { const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldEnabledFalseMapping)); + expect(mappings).toEqual(objectFieldEnabledFalseMapping); }); test('tests processing object field with dynamic set to false', () => { @@ -217,7 +217,7 @@ test('tests processing object field with dynamic set to false', () => { const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicFalseMapping)); + expect(mappings).toEqual(objectFieldDynamicFalseMapping); }); test('tests processing object field with dynamic set to true', () => { @@ -237,7 +237,7 @@ test('tests processing object field with dynamic set to true', () => { const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicTrueMapping)); + expect(mappings).toEqual(objectFieldDynamicTrueMapping); }); test('tests processing object field with dynamic set to strict', () => { @@ -257,7 +257,7 @@ test('tests processing object field with dynamic set to strict', () => { const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicStrictMapping)); + expect(mappings).toEqual(objectFieldDynamicStrictMapping); }); test('tests processing object field with property', () => { @@ -282,7 +282,7 @@ test('tests processing object field with property', () => { const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyMapping)); + expect(mappings).toEqual(objectFieldWithPropertyMapping); }); test('tests processing object field with property, reverse order', () => { @@ -291,10 +291,12 @@ test('tests processing object field with property, reverse order', () => { type: keyword - name: a type: object + dynamic: false `; const objectFieldWithPropertyReversedMapping = { properties: { a: { + dynamic: false, properties: { b: { ignore_above: 1024, @@ -307,7 +309,91 @@ test('tests processing object field with property, reverse order', () => { const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyReversedMapping)); + expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); +}); + +test('tests processing nested field with property', () => { + const nestedYaml = ` + - name: a.b + type: keyword + - name: a + type: nested + dynamic: false + `; + const expectedMapping = { + properties: { + a: { + dynamic: false, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); +}); + +test('tests processing nested field with property, nested field first', () => { + const nestedYaml = ` + - name: a + type: nested + include_in_parent: true + - name: a.b + type: keyword + `; + const expectedMapping = { + properties: { + a: { + include_in_parent: true, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); +}); + +test('tests processing nested leaf field with properties', () => { + const nestedYaml = ` + - name: a + type: object + dynamic: false + - name: a.b + type: nested + enabled: false + `; + const expectedMapping = { + properties: { + a: { + dynamic: false, + properties: { + b: { + enabled: false, + type: 'nested', + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); }); test('tests constant_keyword field type handling', () => { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index 9736f6d1cbd3c..c45c7e706be58 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -71,7 +71,14 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings { switch (type) { case 'group': - fieldProps = generateMappings(field.fields!); + fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) }; + break; + case 'group-nested': + fieldProps = { + ...generateMappings(field.fields!), + ...generateNestedProps(field), + type: 'nested', + }; break; case 'integer': fieldProps.type = 'long'; @@ -95,13 +102,10 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings { } break; case 'object': - fieldProps.type = 'object'; - if (field.hasOwnProperty('enabled')) { - fieldProps.enabled = field.enabled; - } - if (field.hasOwnProperty('dynamic')) { - fieldProps.dynamic = field.dynamic; - } + fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' }; + break; + case 'nested': + fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' }; break; case 'array': // this assumes array fields were validated in an earlier step @@ -128,6 +132,29 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings { return { properties: props }; } +function generateDynamicAndEnabled(field: Field) { + const props: Properties = {}; + if (field.hasOwnProperty('enabled')) { + props.enabled = field.enabled; + } + if (field.hasOwnProperty('dynamic')) { + props.dynamic = field.dynamic; + } + return props; +} + +function generateNestedProps(field: Field) { + const props = generateDynamicAndEnabled(field); + + if (field.hasOwnProperty('include_in_parent')) { + props.include_in_parent = field.include_in_parent; + } + if (field.hasOwnProperty('include_in_root')) { + props.include_in_root = field.include_in_root; + } + return props; +} + function generateMultiFields(fields: Fields): MultiFields { const multiFields: MultiFields = {}; if (fields) { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts index 42989bb1e3ac9..f0ff4c6125452 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.test.ts @@ -210,4 +210,166 @@ describe('processFields', () => { JSON.stringify(objectFieldWithPropertyExpanded) ); }); + + test('correctly handles properties of object type fields where object comes second', () => { + const nested = [ + { + name: 'a.b', + type: 'keyword', + }, + { + name: 'a', + type: 'object', + dynamic: true, + }, + ]; + + const nestedExpanded = [ + { + name: 'a', + type: 'group', + dynamic: true, + fields: [ + { + name: 'b', + type: 'keyword', + }, + ], + }, + ]; + expect(processFields(nested)).toEqual(nestedExpanded); + }); + + test('correctly handles properties of nested type fields', () => { + const nested = [ + { + name: 'a', + type: 'nested', + dynamic: true, + }, + { + name: 'a.b', + type: 'keyword', + }, + ]; + + const nestedExpanded = [ + { + name: 'a', + type: 'group-nested', + dynamic: true, + fields: [ + { + name: 'b', + type: 'keyword', + }, + ], + }, + ]; + expect(processFields(nested)).toEqual(nestedExpanded); + }); + + test('correctly handles properties of nested type where nested top level comes second', () => { + const nested = [ + { + name: 'a.b', + type: 'keyword', + }, + { + name: 'a', + type: 'nested', + dynamic: true, + }, + ]; + + const nestedExpanded = [ + { + name: 'a', + type: 'group-nested', + dynamic: true, + fields: [ + { + name: 'b', + type: 'keyword', + }, + ], + }, + ]; + expect(processFields(nested)).toEqual(nestedExpanded); + }); + + test('ignores redefinitions of an object field', () => { + const object = [ + { + name: 'a', + type: 'object', + dynamic: true, + }, + { + name: 'a', + type: 'object', + dynamic: false, + }, + ]; + + const objectExpected = [ + { + name: 'a', + type: 'object', + // should preserve the field that was parsed first which had dynamic: true + dynamic: true, + }, + ]; + expect(processFields(object)).toEqual(objectExpected); + }); + + test('ignores redefinitions of a nested field', () => { + const nested = [ + { + name: 'a', + type: 'nested', + dynamic: true, + }, + { + name: 'a', + type: 'nested', + dynamic: false, + }, + ]; + + const nestedExpected = [ + { + name: 'a', + type: 'nested', + // should preserve the field that was parsed first which had dynamic: true + dynamic: true, + }, + ]; + expect(processFields(nested)).toEqual(nestedExpected); + }); + + test('ignores redefinitions of a nested and object field', () => { + const nested = [ + { + name: 'a', + type: 'nested', + dynamic: true, + }, + { + name: 'a', + type: 'object', + dynamic: false, + }, + ]; + + const nestedExpected = [ + { + name: 'a', + type: 'nested', + // should preserve the field that was parsed first which had dynamic: true + dynamic: true, + }, + ]; + expect(processFields(nested)).toEqual(nestedExpected); + }); }); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts index edf7624d3f0d5..abaf7ab5b0dfc 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts @@ -28,6 +28,8 @@ export interface Field { object_type?: string; scaling_factor?: number; dynamic?: 'strict' | boolean; + include_in_parent?: boolean; + include_in_root?: boolean; // Kibana specific analyzed?: boolean; @@ -108,18 +110,54 @@ function dedupFields(fields: Fields): Fields { return f.name === field.name; }); if (found) { + // remove name, type, and fields from `field` variable so we avoid merging them into `found` + const { name, type, fields: nestedFields, ...importantFieldProps } = field; + /** + * There are a couple scenarios this if is trying to account for: + * Example 1 + * - name: a.b + * - name: a + * In this scenario found will be `group` and field could be either `object` or `nested` + * Example 2 + * - name: a + * - name: a.b + * In this scenario found could be `object` or `nested` and field will be group + */ if ( - (found.type === 'group' || found.type === 'object') && - field.type === 'group' && - field.fields + // only merge if found is a group and field is object, nested, or group. + // Or if found is object, or nested, and field is a group. + // This is to avoid merging two objects, or nested, or object with a nested. + (found.type === 'group' && + (field.type === 'object' || field.type === 'nested' || field.type === 'group')) || + ((found.type === 'object' || found.type === 'nested') && field.type === 'group') ) { - if (!found.fields) { - found.fields = []; + // if the new field has properties let's dedup and concat them with the already existing found variable in + // the array + if (field.fields) { + // if the found type was object or nested it won't have a fields array so let's initialize it + if (!found.fields) { + found.fields = []; + } + found.fields = dedupFields(found.fields.concat(field.fields)); } - found.type = 'group'; - found.fields = dedupFields(found.fields.concat(field.fields)); + + // if found already had fields or got new ones from the new field coming in we need to assign the right + // type to it + if (found.fields) { + // If this field is supposed to be `nested` and we have fields, we need to preserve the fact that it is + // supposed to be `nested` for when the template is actually generated + if (found.type === 'nested' || field.type === 'nested') { + found.type = 'group-nested'; + } else { + // found was either `group` already or `object` so just set it to `group` + found.type = 'group'; + } + } + // we need to merge in other properties (like `dynamic`) that might exist + Object.assign(found, importantFieldProps); + // if `field.type` wasn't group object or nested, then there's a conflict in types, so lets ignore it } else { - // only 'group' fields can be merged in this way + // only `group`, `object`, and `nested` fields can be merged in this way // XXX: don't abort on error for now // see discussion in https://github.com/elastic/kibana/pull/59894 // throw new Error(