From eec520861937799700f323252b9d6daf7efaa0b0 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Thu, 23 Nov 2023 16:32:48 +0100 Subject: [PATCH] [8.11][Fleet] only add time_series_metric if tsdb enabled (#171712) (#171828) Backport https://github.com/elastic/kibana/pull/171712 to 8.11 --- .../epm/elasticsearch/template/install.ts | 2 +- .../elasticsearch/template/template.test.ts | 64 ++++++++- .../epm/elasticsearch/template/template.ts | 126 ++++++++++-------- .../epm/install_dynamic_template_metric.ts | 8 +- .../0.2.0/data_stream/test/fields/fields.yml | 7 + 5 files changed, 143 insertions(+), 64 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index ea7ccc88f2a4f..48649e8745959 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -534,7 +534,7 @@ export function prepareTemplate({ const validFields = processFields(fields); - const mappings = generateMappings(validFields); + const mappings = generateMappings(validFields, isIndexModeTimeSeries); const templateName = generateTemplateName(dataStream); const templateIndexPattern = generateTemplateIndexPattern(dataStream); const templatePriority = getTemplatePriority(dataStream); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index e5c323de62dfd..1cd83ebf7fb3c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -851,7 +851,30 @@ describe('EPM template', () => { }; const fields: Field[] = safeLoad(literalYml); const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); + const mappings = generateMappings(processedFields, true); + expect(mappings).toEqual(expectedMapping); + }); + + it('tests processing dimension field on a keyword - tsdb disabled', () => { + const literalYml = ` +- name: example.id + type: keyword + dimension: true + `; + const expectedMapping = { + properties: { + example: { + properties: { + id: { + type: 'keyword', + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(literalYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields, false); expect(mappings).toEqual(expectedMapping); }); @@ -875,7 +898,7 @@ describe('EPM template', () => { }; const fields: Field[] = safeLoad(literalYml); const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); + const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(expectedMapping); }); @@ -909,7 +932,40 @@ describe('EPM template', () => { }; const fields: Field[] = safeLoad(literalYml); const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); + const mappings = generateMappings(processedFields, true); + expect(mappings).toEqual(expectedMapping); + }); + + it('tests processing metric_type field - tsdb disabled', () => { + const literalYml = ` +- name: total.norm.pct + type: scaled_float + metric_type: gauge + unit: percent + format: percent +`; + const expectedMapping = { + properties: { + total: { + properties: { + norm: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + meta: { + unit: 'percent', + }, + }, + }, + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(literalYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields, false); expect(mappings).toEqual(expectedMapping); }); @@ -936,7 +992,7 @@ describe('EPM template', () => { }; const fields: Field[] = safeLoad(literalYml); const processedFields = processFields(fields); - const mappings = generateMappings(processedFields); + const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(expectedMapping); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index e55cad3f80a9c..85c814a7e05a1 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -119,46 +119,53 @@ export function getTemplate({ * * @param fields */ -export function generateMappings(fields: Field[]): IndexTemplateMappings { +export function generateMappings( + fields: Field[], + isIndexModeTimeSeries = false +): IndexTemplateMappings { const dynamicTemplates: Array> = []; const dynamicTemplateNames = new Set(); const runtimeFields: RuntimeFields = {}; - const { properties } = _generateMappings(fields, { - addDynamicMapping: (dynamicMapping: { - path: string; - matchingType: string; - pathMatch: string; - properties: string; - runtimeProperties?: string; - }) => { - const name = dynamicMapping.path; - if (dynamicTemplateNames.has(name)) { - return; - } + const { properties } = _generateMappings( + fields, + { + addDynamicMapping: (dynamicMapping: { + path: string; + matchingType: string; + pathMatch: string; + properties: string; + runtimeProperties?: string; + }) => { + const name = dynamicMapping.path; + if (dynamicTemplateNames.has(name)) { + return; + } - const dynamicTemplate: Properties = {}; - if (dynamicMapping.runtimeProperties !== undefined) { - dynamicTemplate.runtime = dynamicMapping.runtimeProperties; - } else { - dynamicTemplate.mapping = dynamicMapping.properties; - } + const dynamicTemplate: Properties = {}; + if (dynamicMapping.runtimeProperties !== undefined) { + dynamicTemplate.runtime = dynamicMapping.runtimeProperties; + } else { + dynamicTemplate.mapping = dynamicMapping.properties; + } - if (dynamicMapping.matchingType) { - dynamicTemplate.match_mapping_type = dynamicMapping.matchingType; - } + if (dynamicMapping.matchingType) { + dynamicTemplate.match_mapping_type = dynamicMapping.matchingType; + } - if (dynamicMapping.pathMatch) { - dynamicTemplate.path_match = dynamicMapping.pathMatch; - } + if (dynamicMapping.pathMatch) { + dynamicTemplate.path_match = dynamicMapping.pathMatch; + } - dynamicTemplateNames.add(name); - dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate }); - }, - addRuntimeField: (runtimeField: { path: string; properties: Properties }) => { - runtimeFields[`${runtimeField.path}`] = runtimeField.properties; + dynamicTemplateNames.add(name); + dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate }); + }, + addRuntimeField: (runtimeField: { path: string; properties: Properties }) => { + runtimeFields[`${runtimeField.path}`] = runtimeField.properties; + }, }, - }); + isIndexModeTimeSeries + ); const indexTemplateMappings: IndexTemplateMappings = { properties }; if (dynamicTemplates.length > 0) { @@ -184,7 +191,8 @@ function _generateMappings( addDynamicMapping: any; addRuntimeField: any; groupFieldName?: string; - } + }, + isIndexModeTimeSeries: boolean ): { properties: IndexTemplateMappings['properties']; hasNonDynamicTemplateMappings: boolean; @@ -206,7 +214,7 @@ function _generateMappings( if (type === 'object' && field.object_type) { const pathMatch = path.includes('*') ? path : `${path}.*`; - let dynProperties: Properties = getDefaultProperties(field); + const dynProperties: Properties = getDefaultProperties(field); let matchingType: string | undefined; switch (field.object_type) { case 'keyword': @@ -216,10 +224,10 @@ function _generateMappings( case 'double': case 'long': case 'boolean': - dynProperties = { - type: field.object_type, - time_series_metric: field.metric_type, - }; + dynProperties.type = field.object_type; + if (isIndexModeTimeSeries) { + dynProperties.time_series_metric = field.metric_type; + } matchingType = field.object_type_mapping_type ?? field.object_type; default: break; @@ -272,10 +280,10 @@ function _generateMappings( case 'long': case 'short': case 'boolean': - dynProperties = { - type: field.object_type, - time_series_metric: field.metric_type, - }; + dynProperties.type = field.object_type; + if (isIndexModeTimeSeries) { + dynProperties.time_series_metric = field.metric_type; + } matchingType = field.object_type_mapping_type ?? field.object_type; default: break; @@ -294,12 +302,16 @@ function _generateMappings( switch (type) { case 'group': - const mappings = _generateMappings(field.fields!, { - ...ctx, - groupFieldName: ctx.groupFieldName - ? `${ctx.groupFieldName}.${field.name}` - : field.name, - }); + const mappings = _generateMappings( + field.fields!, + { + ...ctx, + groupFieldName: ctx.groupFieldName + ? `${ctx.groupFieldName}.${field.name}` + : field.name, + }, + isIndexModeTimeSeries + ); if (!mappings.hasNonDynamicTemplateMappings) { return; } @@ -311,12 +323,16 @@ function _generateMappings( break; case 'group-nested': fieldProps = { - properties: _generateMappings(field.fields!, { - ...ctx, - groupFieldName: ctx.groupFieldName - ? `${ctx.groupFieldName}.${field.name}` - : field.name, - }).properties, + properties: _generateMappings( + field.fields!, + { + ...ctx, + groupFieldName: ctx.groupFieldName + ? `${ctx.groupFieldName}.${field.name}` + : field.name, + }, + isIndexModeTimeSeries + ).properties, ...generateNestedProps(field), type: 'nested', }; @@ -404,10 +420,10 @@ function _generateMappings( } } - if ('metric_type' in field) { + if ('metric_type' in field && isIndexModeTimeSeries) { fieldProps.time_series_metric = field.metric_type; } - if (field.dimension) { + if (field.dimension && isIndexModeTimeSeries) { fieldProps.time_series_dimension = field.dimension; } diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_dynamic_template_metric.ts b/x-pack/test/fleet_api_integration/apis/epm/install_dynamic_template_metric.ts index cc2bb4ebd8582..9bcf3fc673eab 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_dynamic_template_metric.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_dynamic_template_metric.ts @@ -24,14 +24,14 @@ export default function (providerContext: FtrProviderContext) { setupFleetAndAgents(providerContext); after(async () => { - await deletePackage('istio', '0.3.3'); + await deletePackage('no_tsdb_to_tsdb', '0.2.0'); }); it('should install with metric_type added as time_series_metric', async function () { - const templateName = 'metrics-istio.istiod_metrics@package'; + const templateName = 'metrics-no_tsdb_to_tsdb.test@package'; await supertest - .post(`/api/fleet/epm/packages/istio/0.3.3`) + .post(`/api/fleet/epm/packages/no_tsdb_to_tsdb/0.2.0`) .set('kbn-xsrf', 'xxxx') .send({ force: true }) .expect(200); @@ -46,7 +46,7 @@ export default function (providerContext: FtrProviderContext) { const template = resp.component_templates[0].component_template; const dynamicTemplates = template.template.mappings.dynamic_templates; - const mappingName = 'istio.istiod.metrics.*.counter'; + const mappingName = 'test.metrics.*.counter'; const counter = dynamicTemplates.find((tmpl: any) => Object.keys(tmpl)[0] === mappingName); expect(counter[mappingName].mapping.time_series_metric).to.eql('counter'); diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/no_tsdb_to_tsdb/0.2.0/data_stream/test/fields/fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/no_tsdb_to_tsdb/0.2.0/data_stream/test/fields/fields.yml index e1fd2db653efc..6cc90f2148d13 100644 --- a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/no_tsdb_to_tsdb/0.2.0/data_stream/test/fields/fields.yml +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/no_tsdb_to_tsdb/0.2.0/data_stream/test/fields/fields.yml @@ -20,3 +20,10 @@ - name: 'some_metric_field' type: integer metric_type: gauge +- name: test.metrics.*.counter + type: object + object_type: double + object_type_mapping_type: "*" + metric_type: counter + description: > + Istiod counter metric