Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[EPM] Adding support for nested fields #64829

Merged
merged 6 commits into from
Apr 30, 2020
Merged

[EPM] Adding support for nested fields #64829

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
66d5783
Allowing nested types to be merged with group
jonathan-buttner Apr 29, 2020
ee07bd0
Adding nested to case and handling other fields
jonathan-buttner Apr 29, 2020
ffb3a34
Cleaing up if logic
jonathan-buttner Apr 29, 2020
4dd19a3
Removing unneeded if statement
jonathan-buttner Apr 29, 2020
5301a3f
Adding nested type to switch and more tests
jonathan-buttner Apr 29, 2020
22b4a69
Keeping functions immutable
jonathan-buttner Apr 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 96 additions & 10 deletions x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ test('tests processing text field with multi fields', () => {
const fields: Field[] = safeLoad(textWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(textWithMultiFieldsMapping));
expect(mappings).toEqual(textWithMultiFieldsMapping);
});

test('tests processing keyword field with multi fields', () => {
Expand Down Expand Up @@ -127,7 +127,7 @@ test('tests processing keyword field with multi fields', () => {
const fields: Field[] = safeLoad(keywordWithMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithMultiFieldsMapping));
expect(mappings).toEqual(keywordWithMultiFieldsMapping);
});

test('tests processing keyword field with multi fields with analyzed text field', () => {
Expand Down Expand Up @@ -159,7 +159,7 @@ test('tests processing keyword field with multi fields with analyzed text field'
const fields: Field[] = safeLoad(keywordWithAnalyzedMultiFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(keywordWithAnalyzedMultiFieldsMapping));
expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping);
});

test('tests processing object field with no other attributes', () => {
Expand All @@ -177,7 +177,7 @@ test('tests processing object field with no other attributes', () => {
const fields: Field[] = safeLoad(objectFieldLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldMapping));
expect(mappings).toEqual(objectFieldMapping);
});

test('tests processing object field with enabled set to false', () => {
Expand All @@ -197,7 +197,7 @@ test('tests processing object field with enabled set to false', () => {
const fields: Field[] = safeLoad(objectFieldEnabledFalseLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldEnabledFalseMapping));
expect(mappings).toEqual(objectFieldEnabledFalseMapping);
});

test('tests processing object field with dynamic set to false', () => {
Expand All @@ -217,7 +217,7 @@ test('tests processing object field with dynamic set to false', () => {
const fields: Field[] = safeLoad(objectFieldDynamicFalseLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicFalseMapping));
expect(mappings).toEqual(objectFieldDynamicFalseMapping);
});

test('tests processing object field with dynamic set to true', () => {
Expand All @@ -237,7 +237,7 @@ test('tests processing object field with dynamic set to true', () => {
const fields: Field[] = safeLoad(objectFieldDynamicTrueLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicTrueMapping));
expect(mappings).toEqual(objectFieldDynamicTrueMapping);
});

test('tests processing object field with dynamic set to strict', () => {
Expand All @@ -257,7 +257,7 @@ test('tests processing object field with dynamic set to strict', () => {
const fields: Field[] = safeLoad(objectFieldDynamicStrictLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldDynamicStrictMapping));
expect(mappings).toEqual(objectFieldDynamicStrictMapping);
});

test('tests processing object field with property', () => {
Expand All @@ -282,7 +282,7 @@ test('tests processing object field with property', () => {
const fields: Field[] = safeLoad(objectFieldWithPropertyLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyMapping));
expect(mappings).toEqual(objectFieldWithPropertyMapping);
});

test('tests processing object field with property, reverse order', () => {
Expand All @@ -291,10 +291,12 @@ test('tests processing object field with property, reverse order', () => {
type: keyword
- name: a
type: object
dynamic: false
`;
const objectFieldWithPropertyReversedMapping = {
properties: {
a: {
dynamic: false,
properties: {
b: {
ignore_above: 1024,
Expand All @@ -307,5 +309,89 @@ test('tests processing object field with property, reverse order', () => {
const fields: Field[] = safeLoad(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(objectFieldWithPropertyReversedMapping));
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

test('tests processing nested field with property', () => {
const nestedYaml = `
- name: a.b
type: keyword
- name: a
type: nested
dynamic: false
`;
const expectedMapping = {
properties: {
a: {
dynamic: false,
type: 'nested',
properties: {
b: {
ignore_above: 1024,
type: 'keyword',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});

test('tests processing nested field with property, nested field first', () => {
const nestedYaml = `
- name: a
type: nested
include_in_parent: true
- name: a.b
type: keyword
`;
const expectedMapping = {
properties: {
a: {
include_in_parent: true,
type: 'nested',
properties: {
b: {
ignore_above: 1024,
type: 'keyword',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});

test('tests processing nested leaf field with properties', () => {
const nestedYaml = `
- name: a
type: object
dynamic: false
- name: a.b
type: nested
enabled: false
`;
const expectedMapping = {
properties: {
a: {
dynamic: false,
properties: {
b: {
enabled: false,
type: 'nested',
},
},
},
},
};
const fields: Field[] = safeLoad(nestedYaml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(expectedMapping);
});
43 changes: 35 additions & 8 deletions x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,14 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {

switch (type) {
case 'group':
fieldProps = generateMappings(field.fields!);
fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) };
break;
case 'group-nested':
fieldProps = {
...generateMappings(field.fields!),
...generateNestedProps(field),
type: 'nested',
};
break;
case 'integer':
fieldProps.type = 'long';
Expand All @@ -95,13 +102,10 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
}
break;
case 'object':
fieldProps.type = 'object';
if (field.hasOwnProperty('enabled')) {
fieldProps.enabled = field.enabled;
}
if (field.hasOwnProperty('dynamic')) {
fieldProps.dynamic = field.dynamic;
}
fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' };
break;
case 'nested':
fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' };
break;
case 'array':
// this assumes array fields were validated in an earlier step
Expand All @@ -128,6 +132,29 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
return { properties: props };
}

function generateDynamicAndEnabled(field: Field) {
const props: Properties = {};
if (field.hasOwnProperty('enabled')) {
props.enabled = field.enabled;
}
if (field.hasOwnProperty('dynamic')) {
props.dynamic = field.dynamic;
}
return props;
}

function generateNestedProps(field: Field) {
const props = generateDynamicAndEnabled(field);

if (field.hasOwnProperty('include_in_parent')) {
props.include_in_parent = field.include_in_parent;
}
if (field.hasOwnProperty('include_in_root')) {
props.include_in_root = field.include_in_root;
}
return props;
}

function generateMultiFields(fields: Fields): MultiFields {
const multiFields: MultiFields = {};
if (fields) {
Expand Down
Loading