Skip to content

Commit

Permalink
[EPM] Adding support for nested fields (#64829) (#64928)
Browse files Browse the repository at this point in the history 
* Allowing nested types to be merged with group

* Adding nested to case and handling other fields

* Cleaing up if logic

* Removing unneeded if statement

* Adding nested type to switch and more tests

* Keeping functions immutable

Co-authored-by: Elastic Machine <[email protected]>
  • Loading branch information
@jonathan-buttner @elasticmachine
jonathan-buttner and elasticmachine authored May 4, 2020
1 parent 2542977 commit fefdd4d
Show file tree
Hide file tree
Showing 4 changed files with 339 additions and 26 deletions.
106 changes: 96 additions & 10 deletions x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ test('tests processing text field with multi fields', () => {
const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(textWithMultiFieldsMapping));
expect(mappings).toEqual(textWithMultiFieldsMapping);
});

test('tests processing keyword field with multi fields', () => {
Expand Down Expand Up @@ -127,7 +127,7 @@ test('tests processing keyword field with multi fields', () => {
const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithMultiFieldsMapping));
expect(mappings).toEqual(keywordWithMultiFieldsMapping);
});

test('tests processing keyword field with multi fields with analyzed text field', () => {
Expand Down Expand Up @@ -159,7 +159,7 @@ test('tests processing keyword field with multi fields with analyzed text field'
const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithAnalyzedMultiFieldsMapping));
expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping);
});

test('tests processing object field with no other attributes', () => {
Expand All @@ -177,7 +177,7 @@ test('tests processing object field with no other attributes', () => {
const fields: Field[] = safeLoad(objectFieldLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldMapping));
expect(mappings).toEqual(objectFieldMapping);
});

test('tests processing object field with enabled set to false', () => {
Expand All @@ -197,7 +197,7 @@ test('tests processing object field with enabled set to false', () => {
const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldEnabledFalseMapping));
expect(mappings).toEqual(objectFieldEnabledFalseMapping);
});

test('tests processing object field with dynamic set to false', () => {
Expand All @@ -217,7 +217,7 @@ test('tests processing object field with dynamic set to false', () => {
const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicFalseMapping));
expect(mappings).toEqual(objectFieldDynamicFalseMapping);
});

test('tests processing object field with dynamic set to true', () => {
Expand All @@ -237,7 +237,7 @@ test('tests processing object field with dynamic set to true', () => {
const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicTrueMapping));
expect(mappings).toEqual(objectFieldDynamicTrueMapping);
});

test('tests processing object field with dynamic set to strict', () => {
Expand All @@ -257,7 +257,7 @@ test('tests processing object field with dynamic set to strict', () => {
const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicStrictMapping));
expect(mappings).toEqual(objectFieldDynamicStrictMapping);
});

test('tests processing object field with property', () => {
Expand All @@ -282,7 +282,7 @@ test('tests processing object field with property', () => {
const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyMapping));
expect(mappings).toEqual(objectFieldWithPropertyMapping);
});

test('tests processing object field with property, reverse order', () => {
Expand All @@ -291,10 +291,12 @@ test('tests processing object field with property, reverse order', () => {
type: keyword
- name: a
type: object
dynamic: false
`;
const objectFieldWithPropertyReversedMapping = {
properties: {
a: {
dynamic: false,
properties: {
b: {
ignore_above: 1024,
Expand All @@ -307,7 +309,91 @@ test('tests processing object field with property, reverse order', () => {
const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyReversedMapping));
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

test('tests processing nested field with property', () => {
const nestedYaml = `
- name: a.b
type: keyword
- name: a
type: nested
dynamic: false
`;
const expectedMapping = {
properties: {
a: {
dynamic: false,
type: 'nested',
properties: {
b: {
ignore_above: 1024,
type: 'keyword',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});

test('tests processing nested field with property, nested field first', () => {
const nestedYaml = `
- name: a
type: nested
include_in_parent: true
- name: a.b
type: keyword
`;
const expectedMapping = {
properties: {
a: {
include_in_parent: true,
type: 'nested',
properties: {
b: {
ignore_above: 1024,
type: 'keyword',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});

test('tests processing nested leaf field with properties', () => {
const nestedYaml = `
- name: a
type: object
dynamic: false
- name: a.b
type: nested
enabled: false
`;
const expectedMapping = {
properties: {
a: {
dynamic: false,
properties: {
b: {
enabled: false,
type: 'nested',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});

test('tests constant_keyword field type handling', () => {
Expand Down
43 changes: 35 additions & 8 deletions x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,14 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {

switch (type) {
case 'group':
fieldProps = generateMappings(field.fields!);
fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) };
break;
case 'group-nested':
fieldProps = {
...generateMappings(field.fields!),
...generateNestedProps(field),
type: 'nested',
};
break;
case 'integer':
fieldProps.type = 'long';
Expand All @@ -95,13 +102,10 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
}
break;
case 'object':
fieldProps.type = 'object';
if (field.hasOwnProperty('enabled')) {
fieldProps.enabled = field.enabled;
}
if (field.hasOwnProperty('dynamic')) {
fieldProps.dynamic = field.dynamic;
}
fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' };
break;
case 'nested':
fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' };
break;
case 'array':
// this assumes array fields were validated in an earlier step
Expand All @@ -128,6 +132,29 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
return { properties: props };
}

function generateDynamicAndEnabled(field: Field) {
const props: Properties = {};
if (field.hasOwnProperty('enabled')) {
props.enabled = field.enabled;
}
if (field.hasOwnProperty('dynamic')) {
props.dynamic = field.dynamic;
}
return props;
}

function generateNestedProps(field: Field) {
const props = generateDynamicAndEnabled(field);

if (field.hasOwnProperty('include_in_parent')) {
props.include_in_parent = field.include_in_parent;
}
if (field.hasOwnProperty('include_in_root')) {
props.include_in_root = field.include_in_root;
}
return props;
}

function generateMultiFields(fields: Fields): MultiFields {
const multiFields: MultiFields = {};
if (fields) {
Expand Down
Loading

0 comments on commit fefdd4d

Please sign in to comment.