From 562a6d859e8641c31cd8a481f2e4262ccbda43bc Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Thu, 2 Nov 2023 09:55:32 +0100 Subject: [PATCH] [Fleet] Use default component templates from Elasticsearch (#163731) ## Summary Fixes https://github.com/elastic/kibana/issues/163141 Fixes #160288 Blocked by: - https://github.com/elastic/elasticsearch/issues/98535 This switches where integrations installed by EPM get their default index settings from to use the [source-of-truth component templates supplied by Elasticsearch](https://github.com/elastic/elasticsearch/tree/main/x-pack/plugin/core/template-resources/src/main/resources). This will help ensure that data streams configured by EPM always get the same defaults as data streams the user creates using the default `logs-*-*` and `metrics-*-*` templates. For now, no default mappings are sourced from Elasticsearch. As part of this change the template format version was incremented to force EPM to reinstall all templates and rollover data streams on the Stack upgrade to the version including this change. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../server/create_indices/create_indices.ts | 11 +++- .../plugins/fleet/common/types/models/epm.ts | 1 + .../fleet/server/constants/fleet_es_assets.ts | 12 ++++- .../plugins/fleet/server/constants/index.ts | 4 ++ .../template/default_settings.test.ts | 4 -- .../template/default_settings.ts | 10 ---- .../elasticsearch/template/install.test.ts | 28 +++++++++- .../epm/elasticsearch/template/install.ts | 6 ++- .../elasticsearch/template/template.test.ts | 54 +++++++++++++++++-- .../epm/elasticsearch/template/template.ts | 22 ++++++++ .../apis/epm/install_overrides.ts | 4 +- .../apis/epm/template.ts | 1 + .../apis/epm/update_assets.ts | 2 - .../input_package_create_upgrade.ts | 2 - 14 files changed, 131 insertions(+), 30 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts index f8a935f361ff6..91ee56c1fbefd 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts @@ -7,6 +7,7 @@ import { errors } from '@elastic/elasticsearch'; import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS } from '@kbn/fleet-plugin/server/constants'; import { BENCHMARK_SCORE_INDEX_DEFAULT_NS, BENCHMARK_SCORE_INDEX_PATTERN, @@ -24,6 +25,10 @@ import { CloudSecurityPostureConfig } from '../config'; interface IndexTemplateSettings { index: { default_pipeline: string; + codec?: string; + mapping?: { + ignore_malformed: boolean; + }; }; lifecycle?: { name: string }; } @@ -226,6 +231,10 @@ const updateIndexTemplate = async ( ...template?.settings, // nothing inside index: { default_pipeline: latestFindingsPipelineIngestConfig.id, + codec: 'best_compression', + mapping: { + ignore_malformed: true, + }, }, lifecycle: { name: '' }, }; @@ -242,7 +251,7 @@ const updateIndexTemplate = async ( aliases: template?.aliases, }, _meta, - composed_of: composedOf, + composed_of: composedOf.filter((ct) => ct !== STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS), }); logger.info(`Updated index template successfully [Name: ${indexTemplateName}]`); diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index c05acb3de8b9f..d8400a8a61b9b 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -615,6 +615,7 @@ export interface IndexTemplate { }; data_stream: { hidden?: boolean }; composed_of: string[]; + ignore_missing_component_templates?: string[]; _meta: object; } diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index c33a53f4dd4a6..b90be40075d8e 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -11,7 +11,7 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta'; const meta = getESAssetMetadata(); -export const FLEET_INSTALL_FORMAT_VERSION = '1.0.0'; +export const FLEET_INSTALL_FORMAT_VERSION = '1.1.0'; export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1'; @@ -81,6 +81,16 @@ export const FLEET_COMPONENT_TEMPLATES = [ }, ]; +export const STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS = `logs@settings`; +export const STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS = `metrics@settings`; +export const STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS = `metrics@tsdb-settings`; + +export const STACK_COMPONENT_TEMPLATES = [ + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS, +]; + export const FLEET_FINAL_PIPELINE_VERSION = 4; // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index 37e570648e392..7a03f7c33ab3d 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -95,6 +95,10 @@ export { FLEET_FINAL_PIPELINE_VERSION, FLEET_INSTALL_FORMAT_VERSION, FLEET_AGENT_POLICIES_SCHEMA_VERSION, + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS, + STACK_COMPONENT_TEMPLATES, } from './fleet_es_assets'; export { FILE_STORAGE_DATA_AGENT_INDEX } from './fleet_es_assets'; export { FILE_STORAGE_METADATA_AGENT_INDEX } from './fleet_es_assets'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts index 3f7fa91b6462c..6452671644763 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts @@ -65,13 +65,9 @@ describe('buildDefaultSettings', () => { expect(settings).toMatchInlineSnapshot(` Object { "index": Object { - "codec": "best_compression", "lifecycle": Object { "name": "logs", }, - "mapping": Object { - "ignore_malformed": true, - }, "query": Object { "default_field": Array [ "field1Keyword", diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts index dc61f4fca9e5c..6d42f106464b4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts @@ -73,16 +73,6 @@ export function buildDefaultSettings({ name: ilmPolicy ? ilmPolicy : type, }, }), - // What should be our default for the compression? - codec: 'best_compression', - // setting `ignore_malformed` only for data_stream for logs - ...(type === 'logs' - ? { - mapping: { - ignore_malformed: true, - }, - } - : {}), // All the default fields which should be queried have to be added here. // So far we add all keyword and text fields here if there are any, otherwise // this setting is skipped. diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index bf15db1151744..321e832115cf0 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -7,9 +7,9 @@ import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; import { loadFieldsFromYaml } from '../../fields/field'; -import type { RegistryDataStream } from '../../../../types'; +import type { ArchivePackage, RegistryDataStream } from '../../../../types'; -import { prepareTemplate } from './install'; +import { prepareTemplate, prepareToInstallTemplates } from './install'; jest.mock('../../fields/field', () => ({ ...jest.requireActual('../../fields/field'), @@ -455,4 +455,28 @@ describe('EPM index template install', () => { expect(packageTemplate).not.toHaveProperty('lifecycle'); }); + + test('test prepareToInstallTemplates does not include stack component templates in tracked assets', () => { + const dataStreamDatasetIsPrefixUnset = { + type: 'logs', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + + const { assetsToAdd } = prepareToInstallTemplates( + { + name: 'package', + version: '0.0.1', + data_streams: [dataStreamDatasetIsPrefixUnset], + } as ArchivePackage, + [], + [] + ); + + expect(assetsToAdd).not.toContainEqual({ id: 'logs@settings', type: 'component_template' }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index ea7ccc88f2a4f..d65d8de5a3828 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -38,6 +38,7 @@ import { FLEET_COMPONENT_TEMPLATES, PACKAGE_TEMPLATE_SUFFIX, USER_SETTINGS_TEMPLATE_SUFFIX, + STACK_COMPONENT_TEMPLATES, } from '../../../../constants'; import { getESAssetMetadata } from '../meta'; @@ -530,7 +531,7 @@ export function prepareTemplate({ const isIndexModeTimeSeries = dataStream.elasticsearch?.index_mode === 'time_series' || - experimentalDataStreamFeature?.features.tsdb; + !!experimentalDataStreamFeature?.features.tsdb; const validFields = processFields(fields); @@ -572,6 +573,7 @@ export function prepareTemplate({ registryElasticsearch: dataStream.elasticsearch, mappings, isIndexModeTimeSeries, + type: dataStream.type, }); return { @@ -616,6 +618,8 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) { .filter( (componentTemplateId) => !FLEET_COMPONENT_TEMPLATE_NAMES.includes(componentTemplateId) ) + // Filter stack component templates shared between integrations + .filter((componentTemplateId) => !STACK_COMPONENT_TEMPLATES.includes(componentTemplateId)) .map((componentTemplateId) => ({ id: componentTemplateId, type: ElasticsearchAssetType.componentTemplate, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index f0459b10c8570..59e5c68fb7345 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -57,10 +57,12 @@ describe('EPM template', () => { const template = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); @@ -69,13 +71,35 @@ describe('EPM template', () => { const composedOfTemplates = ['component1', 'component2']; const template = getTemplate({ - templateIndexPattern: 'name-*', + templateIndexPattern: 'logs-*', + type: 'logs', packageName: 'nginx', composedOfTemplates, templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(template.composed_of).toStrictEqual([ + 'logs@settings', + ...composedOfTemplates, + ...FLEET_COMPONENT_TEMPLATES_NAMES, + ]); + }); + + it('supplies metrics@tsdb-settings for time series', () => { + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + templateIndexPattern: 'metrics-*', + type: 'metrics', + packageName: 'nginx', + composedOfTemplates, + templatePriority: 200, + mappings: { properties: [] }, + isIndexModeTimeSeries: true, + }); + expect(template.composed_of).toStrictEqual([ + 'metrics@tsdb-settings', ...composedOfTemplates, ...FLEET_COMPONENT_TEMPLATES_NAMES, ]); @@ -90,13 +114,16 @@ describe('EPM template', () => { const composedOfTemplates = ['component1', 'component2']; const template = getTemplate({ - templateIndexPattern: 'name-*', + templateIndexPattern: 'logs-*', + type: 'logs', packageName: 'nginx', composedOfTemplates, templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(template.composed_of).toStrictEqual([ + 'logs@settings', ...composedOfTemplates, FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, ]); @@ -106,13 +133,18 @@ describe('EPM template', () => { const composedOfTemplates: string[] = []; const template = getTemplate({ - templateIndexPattern: 'name-*', + templateIndexPattern: 'logs-*', + type: 'logs', packageName: 'nginx', composedOfTemplates, templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); - expect(template.composed_of).toStrictEqual(FLEET_COMPONENT_TEMPLATES_NAMES); + expect(template.composed_of).toStrictEqual([ + 'logs@settings', + ...FLEET_COMPONENT_TEMPLATES_NAMES, + ]); }); it('adds hidden field correctly', () => { @@ -120,20 +152,24 @@ describe('EPM template', () => { const templateWithHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, hidden: true, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(templateWithHidden.data_stream.hidden).toEqual(true); const templateWithoutHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); @@ -143,6 +179,7 @@ describe('EPM template', () => { const templateWithGlobalAndDataStreamHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, @@ -153,11 +190,13 @@ describe('EPM template', () => { hidden: true, }, }, + isIndexModeTimeSeries: false, }); expect(templateWithGlobalAndDataStreamHidden.data_stream.hidden).toEqual(true); const templateWithDataStreamHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, @@ -167,21 +206,25 @@ describe('EPM template', () => { hidden: true, }, }, + isIndexModeTimeSeries: false, }); expect(templateWithDataStreamHidden.data_stream.hidden).toEqual(true); const templateWithoutDataStreamHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, hidden: true, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(templateWithoutDataStreamHidden.data_stream.hidden).toEqual(true); const templateWithGlobalHiddenTrueAndDataStreamHiddenFalse = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, @@ -192,15 +235,18 @@ describe('EPM template', () => { hidden: false, }, }, + isIndexModeTimeSeries: false, }); expect(templateWithGlobalHiddenTrueAndDataStreamHiddenFalse.data_stream.hidden).toEqual(true); const templateWithoutHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 36e13ae331f58..26c6926c0bbc4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -27,6 +27,9 @@ import { getRegistryDataStreamAssetBaseName } from '../../../../../common/servic import { FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS, } from '../../../../constants'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -76,12 +79,14 @@ export function getTemplate({ registryElasticsearch, mappings, isIndexModeTimeSeries, + type, }: { templateIndexPattern: string; packageName: string; composedOfTemplates: string[]; templatePriority: number; mappings: IndexTemplateMappings; + type: string; hidden?: boolean; registryElasticsearch?: RegistryElasticsearch | undefined; isIndexModeTimeSeries?: boolean; @@ -100,7 +105,10 @@ export function getTemplate({ throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`); } + const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries); + template.composed_of = [ + ...esBaseComponents, ...(template.composed_of || []), FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, ...(appContextService.getConfig()?.agentIdVerificationEnabled @@ -111,6 +119,20 @@ export function getTemplate({ return template; } +const getBaseEsComponents = (type: string, isIndexModeTimeSeries: boolean): string[] => { + if (type === 'metrics') { + if (isIndexModeTimeSeries) { + return [STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS]; + } + + return [STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS]; + } else if (type === 'logs') { + return [STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS]; + } + + return []; +}; + /** * Generate mapping takes the given nested fields array and creates the Elasticsearch * mapping properties out of it. diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index c93f30dddff7e..26211028a3411 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -57,6 +57,7 @@ export default function (providerContext: FtrProviderContext) { // the index template composed_of has the correct component templates in the correct order const indexTemplate = indexTemplateResponse.index_templates[0].index_template; expect(indexTemplate.composed_of).to.eql([ + `logs@settings`, `${templateName}@package`, `${templateName}@custom`, '.fleet_globals-1', @@ -131,13 +132,11 @@ export default function (providerContext: FtrProviderContext) { template: { settings: { index: { - codec: 'best_compression', default_pipeline: 'logs-overrides.test-0.1.0', lifecycle: { name: 'overridden by user', }, mapping: { - ignore_malformed: `true`, total_fields: { limit: '10000', }, @@ -149,7 +148,6 @@ export default function (providerContext: FtrProviderContext) { dynamic: 'false', properties: { '@timestamp': { - ignore_malformed: false, type: 'date', }, data_stream: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index c62afa6dccbf7..8f2d208a27421 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -35,6 +35,7 @@ export default function ({ getService }: FtrProviderContext) { composedOfTemplates: [], templatePriority: 200, mappings: { properties: [] }, + type: 'logs', }); // This test is not an API integration test with Kibana diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 9b2527a5b025d..75040c8400d04 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -217,13 +217,11 @@ export default function (providerContext: FtrProviderContext) { expect(resPackage.statusCode).equal(200); expect(resPackage.body.component_templates[0].component_template.template.settings).eql({ index: { - codec: 'best_compression', default_pipeline: 'logs-all_assets.test_logs-0.2.0', lifecycle: { name: 'reference2', }, mapping: { - ignore_malformed: `true`, total_fields: { limit: '10000', }, diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts b/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts index a1e6fe5d5556f..0d2af1d0e6ac1 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts @@ -221,11 +221,9 @@ export default function (providerContext: FtrProviderContext) { settings: { index: { lifecycle: { name: 'logs' }, - codec: 'best_compression', default_pipeline: 'logs-dataset1-1.0.0', mapping: { total_fields: { limit: '10000' }, - ignore_malformed: 'true', }, }, },