Skip to content

Commit

Permalink
[8.11][Fleet] only add time_series_metric if tsdb enabled (#171712) (#…
Browse files Browse the repository at this point in the history 
…171828)

Backport #171712 to 8.11
  • Loading branch information
@juliaElastic
juliaElastic authored Nov 23, 2023
1 parent 7f1b79c commit eec5208
Show file tree
Hide file tree
Showing 5 changed files with 143 additions and 64 deletions.
2 changes: 1 addition & 1 deletion x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,7 @@ export function prepareTemplate({

const validFields = processFields(fields);

const mappings = generateMappings(validFields);
const mappings = generateMappings(validFields, isIndexModeTimeSeries);
const templateName = generateTemplateName(dataStream);
const templateIndexPattern = generateTemplateIndexPattern(dataStream);
const templatePriority = getTemplatePriority(dataStream);
Expand Down
64 changes: 60 additions & 4 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -851,7 +851,30 @@ describe('EPM template', () => {
};
const fields: Field[] = safeLoad(literalYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
const mappings = generateMappings(processedFields, true);
expect(mappings).toEqual(expectedMapping);
});

it('tests processing dimension field on a keyword - tsdb disabled', () => {
const literalYml = `
- name: example.id
type: keyword
dimension: true
`;
const expectedMapping = {
properties: {
example: {
properties: {
id: {
type: 'keyword',
},
},
},
},
};
const fields: Field[] = safeLoad(literalYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields, false);
expect(mappings).toEqual(expectedMapping);
});

Expand All @@ -875,7 +898,7 @@ describe('EPM template', () => {
};
const fields: Field[] = safeLoad(literalYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
const mappings = generateMappings(processedFields, true);
expect(mappings).toEqual(expectedMapping);
});

Expand Down Expand Up @@ -909,7 +932,40 @@ describe('EPM template', () => {
};
const fields: Field[] = safeLoad(literalYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
const mappings = generateMappings(processedFields, true);
expect(mappings).toEqual(expectedMapping);
});

it('tests processing metric_type field - tsdb disabled', () => {
const literalYml = `
- name: total.norm.pct
type: scaled_float
metric_type: gauge
unit: percent
format: percent
`;
const expectedMapping = {
properties: {
total: {
properties: {
norm: {
properties: {
pct: {
scaling_factor: 1000,
type: 'scaled_float',
meta: {
unit: 'percent',
},
},
},
},
},
},
},
};
const fields: Field[] = safeLoad(literalYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields, false);
expect(mappings).toEqual(expectedMapping);
});

Expand All @@ -936,7 +992,7 @@ describe('EPM template', () => {
};
const fields: Field[] = safeLoad(literalYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
const mappings = generateMappings(processedFields, true);
expect(mappings).toEqual(expectedMapping);
});

Expand Down
126 changes: 71 additions & 55 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,46 +119,53 @@ export function getTemplate({
*
* @param fields
*/
export function generateMappings(fields: Field[]): IndexTemplateMappings {
export function generateMappings(
fields: Field[],
isIndexModeTimeSeries = false
): IndexTemplateMappings {
const dynamicTemplates: Array<Record<string, Properties>> = [];
const dynamicTemplateNames = new Set<string>();
const runtimeFields: RuntimeFields = {};

const { properties } = _generateMappings(fields, {
addDynamicMapping: (dynamicMapping: {
path: string;
matchingType: string;
pathMatch: string;
properties: string;
runtimeProperties?: string;
}) => {
const name = dynamicMapping.path;
if (dynamicTemplateNames.has(name)) {
return;
}
const { properties } = _generateMappings(
fields,
{
addDynamicMapping: (dynamicMapping: {
path: string;
matchingType: string;
pathMatch: string;
properties: string;
runtimeProperties?: string;
}) => {
const name = dynamicMapping.path;
if (dynamicTemplateNames.has(name)) {
return;
}

const dynamicTemplate: Properties = {};
if (dynamicMapping.runtimeProperties !== undefined) {
dynamicTemplate.runtime = dynamicMapping.runtimeProperties;
} else {
dynamicTemplate.mapping = dynamicMapping.properties;
}
const dynamicTemplate: Properties = {};
if (dynamicMapping.runtimeProperties !== undefined) {
dynamicTemplate.runtime = dynamicMapping.runtimeProperties;
} else {
dynamicTemplate.mapping = dynamicMapping.properties;
}

if (dynamicMapping.matchingType) {
dynamicTemplate.match_mapping_type = dynamicMapping.matchingType;
}
if (dynamicMapping.matchingType) {
dynamicTemplate.match_mapping_type = dynamicMapping.matchingType;
}

if (dynamicMapping.pathMatch) {
dynamicTemplate.path_match = dynamicMapping.pathMatch;
}
if (dynamicMapping.pathMatch) {
dynamicTemplate.path_match = dynamicMapping.pathMatch;
}

dynamicTemplateNames.add(name);
dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate });
},
addRuntimeField: (runtimeField: { path: string; properties: Properties }) => {
runtimeFields[`${runtimeField.path}`] = runtimeField.properties;
dynamicTemplateNames.add(name);
dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate });
},
addRuntimeField: (runtimeField: { path: string; properties: Properties }) => {
runtimeFields[`${runtimeField.path}`] = runtimeField.properties;
},
},
});
isIndexModeTimeSeries
);

const indexTemplateMappings: IndexTemplateMappings = { properties };
if (dynamicTemplates.length > 0) {
Expand All @@ -184,7 +191,8 @@ function _generateMappings(
addDynamicMapping: any;
addRuntimeField: any;
groupFieldName?: string;
}
},
isIndexModeTimeSeries: boolean
): {
properties: IndexTemplateMappings['properties'];
hasNonDynamicTemplateMappings: boolean;
Expand All @@ -206,7 +214,7 @@ function _generateMappings(
if (type === 'object' && field.object_type) {
const pathMatch = path.includes('*') ? path : `${path}.*`;

let dynProperties: Properties = getDefaultProperties(field);
const dynProperties: Properties = getDefaultProperties(field);
let matchingType: string | undefined;
switch (field.object_type) {
case 'keyword':
Expand All @@ -216,10 +224,10 @@ function _generateMappings(
case 'double':
case 'long':
case 'boolean':
dynProperties = {
type: field.object_type,
time_series_metric: field.metric_type,
};
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? field.object_type;
default:
break;
Expand Down Expand Up @@ -272,10 +280,10 @@ function _generateMappings(
case 'long':
case 'short':
case 'boolean':
dynProperties = {
type: field.object_type,
time_series_metric: field.metric_type,
};
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? field.object_type;
default:
break;
Expand All @@ -294,12 +302,16 @@ function _generateMappings(

switch (type) {
case 'group':
const mappings = _generateMappings(field.fields!, {
...ctx,
groupFieldName: ctx.groupFieldName
? `${ctx.groupFieldName}.${field.name}`
: field.name,
});
const mappings = _generateMappings(
field.fields!,
{
...ctx,
groupFieldName: ctx.groupFieldName
? `${ctx.groupFieldName}.${field.name}`
: field.name,
},
isIndexModeTimeSeries
);
if (!mappings.hasNonDynamicTemplateMappings) {
return;
}
Expand All @@ -311,12 +323,16 @@ function _generateMappings(
break;
case 'group-nested':
fieldProps = {
properties: _generateMappings(field.fields!, {
...ctx,
groupFieldName: ctx.groupFieldName
? `${ctx.groupFieldName}.${field.name}`
: field.name,
}).properties,
properties: _generateMappings(
field.fields!,
{
...ctx,
groupFieldName: ctx.groupFieldName
? `${ctx.groupFieldName}.${field.name}`
: field.name,
},
isIndexModeTimeSeries
).properties,
...generateNestedProps(field),
type: 'nested',
};
Expand Down Expand Up @@ -404,10 +420,10 @@ function _generateMappings(
}
}

if ('metric_type' in field) {
if ('metric_type' in field && isIndexModeTimeSeries) {
fieldProps.time_series_metric = field.metric_type;
}
if (field.dimension) {
if (field.dimension && isIndexModeTimeSeries) {
fieldProps.time_series_dimension = field.dimension;
}

Expand Down
8 changes: 4 additions & 4 deletions x-pack/test/fleet_api_integration/apis/epm/install_dynamic_template_metric.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ export default function (providerContext: FtrProviderContext) {
setupFleetAndAgents(providerContext);

after(async () => {
await deletePackage('istio', '0.3.3');
await deletePackage('no_tsdb_to_tsdb', '0.2.0');
});

it('should install with metric_type added as time_series_metric', async function () {
const templateName = 'metrics-istio.istiod_metrics@package';
const templateName = 'metrics-no_tsdb_to_tsdb.test@package';

await supertest
.post(`/api/fleet/epm/packages/istio/0.3.3`)
.post(`/api/fleet/epm/packages/no_tsdb_to_tsdb/0.2.0`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true })
.expect(200);
Expand All @@ -46,7 +46,7 @@ export default function (providerContext: FtrProviderContext) {

const template = resp.component_templates[0].component_template;
const dynamicTemplates = template.template.mappings.dynamic_templates;
const mappingName = 'istio.istiod.metrics.*.counter';
const mappingName = 'test.metrics.*.counter';
const counter = dynamicTemplates.find((tmpl: any) => Object.keys(tmpl)[0] === mappingName);

expect(counter[mappingName].mapping.time_series_metric).to.eql('counter');
Expand Down
7 changes: 7 additions & 0 deletions ...tion/apis/fixtures/test_packages/no_tsdb_to_tsdb/0.2.0/data_stream/test/fields/fields.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,10 @@
- name: 'some_metric_field'
type: integer
metric_type: gauge
- name: test.metrics.*.counter
type: object
object_type: double
object_type_mapping_type: "*"
metric_type: counter
description: >
Istiod counter metric

0 comments on commit eec5208

Please sign in to comment.