diff --git a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts index f8a935f361ff6..91ee56c1fbefd 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts @@ -7,6 +7,7 @@ import { errors } from '@elastic/elasticsearch'; import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS } from '@kbn/fleet-plugin/server/constants'; import { BENCHMARK_SCORE_INDEX_DEFAULT_NS, BENCHMARK_SCORE_INDEX_PATTERN, @@ -24,6 +25,10 @@ import { CloudSecurityPostureConfig } from '../config'; interface IndexTemplateSettings { index: { default_pipeline: string; + codec?: string; + mapping?: { + ignore_malformed: boolean; + }; }; lifecycle?: { name: string }; } @@ -226,6 +231,10 @@ const updateIndexTemplate = async ( ...template?.settings, // nothing inside index: { default_pipeline: latestFindingsPipelineIngestConfig.id, + codec: 'best_compression', + mapping: { + ignore_malformed: true, + }, }, lifecycle: { name: '' }, }; @@ -242,7 +251,7 @@ const updateIndexTemplate = async ( aliases: template?.aliases, }, _meta, - composed_of: composedOf, + composed_of: composedOf.filter((ct) => ct !== STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS), }); logger.info(`Updated index template successfully [Name: ${indexTemplateName}]`); diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index c05acb3de8b9f..d8400a8a61b9b 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -615,6 +615,7 @@ export interface IndexTemplate { }; data_stream: { hidden?: boolean }; composed_of: string[]; + ignore_missing_component_templates?: string[]; _meta: object; } diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index c33a53f4dd4a6..b90be40075d8e 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -11,7 +11,7 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta'; const meta = getESAssetMetadata(); -export const FLEET_INSTALL_FORMAT_VERSION = '1.0.0'; +export const FLEET_INSTALL_FORMAT_VERSION = '1.1.0'; export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1'; @@ -81,6 +81,16 @@ export const FLEET_COMPONENT_TEMPLATES = [ }, ]; +export const STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS = `logs@settings`; +export const STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS = `metrics@settings`; +export const STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS = `metrics@tsdb-settings`; + +export const STACK_COMPONENT_TEMPLATES = [ + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS, +]; + export const FLEET_FINAL_PIPELINE_VERSION = 4; // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index 37e570648e392..7a03f7c33ab3d 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -95,6 +95,10 @@ export { FLEET_FINAL_PIPELINE_VERSION, FLEET_INSTALL_FORMAT_VERSION, FLEET_AGENT_POLICIES_SCHEMA_VERSION, + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS, + STACK_COMPONENT_TEMPLATES, } from './fleet_es_assets'; export { FILE_STORAGE_DATA_AGENT_INDEX } from './fleet_es_assets'; export { FILE_STORAGE_METADATA_AGENT_INDEX } from './fleet_es_assets'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts index 3f7fa91b6462c..6452671644763 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts @@ -65,13 +65,9 @@ describe('buildDefaultSettings', () => { expect(settings).toMatchInlineSnapshot(` Object { "index": Object { - "codec": "best_compression", "lifecycle": Object { "name": "logs", }, - "mapping": Object { - "ignore_malformed": true, - }, "query": Object { "default_field": Array [ "field1Keyword", diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts index dc61f4fca9e5c..6d42f106464b4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts @@ -73,16 +73,6 @@ export function buildDefaultSettings({ name: ilmPolicy ? ilmPolicy : type, }, }), - // What should be our default for the compression? - codec: 'best_compression', - // setting `ignore_malformed` only for data_stream for logs - ...(type === 'logs' - ? { - mapping: { - ignore_malformed: true, - }, - } - : {}), // All the default fields which should be queried have to be added here. // So far we add all keyword and text fields here if there are any, otherwise // this setting is skipped. diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index bf15db1151744..321e832115cf0 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -7,9 +7,9 @@ import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; import { loadFieldsFromYaml } from '../../fields/field'; -import type { RegistryDataStream } from '../../../../types'; +import type { ArchivePackage, RegistryDataStream } from '../../../../types'; -import { prepareTemplate } from './install'; +import { prepareTemplate, prepareToInstallTemplates } from './install'; jest.mock('../../fields/field', () => ({ ...jest.requireActual('../../fields/field'), @@ -455,4 +455,28 @@ describe('EPM index template install', () => { expect(packageTemplate).not.toHaveProperty('lifecycle'); }); + + test('test prepareToInstallTemplates does not include stack component templates in tracked assets', () => { + const dataStreamDatasetIsPrefixUnset = { + type: 'logs', + dataset: 'package.dataset', + title: 'test data stream', + release: 'experimental', + package: 'package', + path: 'path', + ingest_pipeline: 'default', + } as RegistryDataStream; + + const { assetsToAdd } = prepareToInstallTemplates( + { + name: 'package', + version: '0.0.1', + data_streams: [dataStreamDatasetIsPrefixUnset], + } as ArchivePackage, + [], + [] + ); + + expect(assetsToAdd).not.toContainEqual({ id: 'logs@settings', type: 'component_template' }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index ea7ccc88f2a4f..d65d8de5a3828 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -38,6 +38,7 @@ import { FLEET_COMPONENT_TEMPLATES, PACKAGE_TEMPLATE_SUFFIX, USER_SETTINGS_TEMPLATE_SUFFIX, + STACK_COMPONENT_TEMPLATES, } from '../../../../constants'; import { getESAssetMetadata } from '../meta'; @@ -530,7 +531,7 @@ export function prepareTemplate({ const isIndexModeTimeSeries = dataStream.elasticsearch?.index_mode === 'time_series' || - experimentalDataStreamFeature?.features.tsdb; + !!experimentalDataStreamFeature?.features.tsdb; const validFields = processFields(fields); @@ -572,6 +573,7 @@ export function prepareTemplate({ registryElasticsearch: dataStream.elasticsearch, mappings, isIndexModeTimeSeries, + type: dataStream.type, }); return { @@ -616,6 +618,8 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) { .filter( (componentTemplateId) => !FLEET_COMPONENT_TEMPLATE_NAMES.includes(componentTemplateId) ) + // Filter stack component templates shared between integrations + .filter((componentTemplateId) => !STACK_COMPONENT_TEMPLATES.includes(componentTemplateId)) .map((componentTemplateId) => ({ id: componentTemplateId, type: ElasticsearchAssetType.componentTemplate, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index f0459b10c8570..59e5c68fb7345 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -57,10 +57,12 @@ describe('EPM template', () => { const template = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(template.index_patterns).toStrictEqual([templateIndexPattern]); }); @@ -69,13 +71,35 @@ describe('EPM template', () => { const composedOfTemplates = ['component1', 'component2']; const template = getTemplate({ - templateIndexPattern: 'name-*', + templateIndexPattern: 'logs-*', + type: 'logs', packageName: 'nginx', composedOfTemplates, templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(template.composed_of).toStrictEqual([ + 'logs@settings', + ...composedOfTemplates, + ...FLEET_COMPONENT_TEMPLATES_NAMES, + ]); + }); + + it('supplies metrics@tsdb-settings for time series', () => { + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + templateIndexPattern: 'metrics-*', + type: 'metrics', + packageName: 'nginx', + composedOfTemplates, + templatePriority: 200, + mappings: { properties: [] }, + isIndexModeTimeSeries: true, + }); + expect(template.composed_of).toStrictEqual([ + 'metrics@tsdb-settings', ...composedOfTemplates, ...FLEET_COMPONENT_TEMPLATES_NAMES, ]); @@ -90,13 +114,16 @@ describe('EPM template', () => { const composedOfTemplates = ['component1', 'component2']; const template = getTemplate({ - templateIndexPattern: 'name-*', + templateIndexPattern: 'logs-*', + type: 'logs', packageName: 'nginx', composedOfTemplates, templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(template.composed_of).toStrictEqual([ + 'logs@settings', ...composedOfTemplates, FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, ]); @@ -106,13 +133,18 @@ describe('EPM template', () => { const composedOfTemplates: string[] = []; const template = getTemplate({ - templateIndexPattern: 'name-*', + templateIndexPattern: 'logs-*', + type: 'logs', packageName: 'nginx', composedOfTemplates, templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); - expect(template.composed_of).toStrictEqual(FLEET_COMPONENT_TEMPLATES_NAMES); + expect(template.composed_of).toStrictEqual([ + 'logs@settings', + ...FLEET_COMPONENT_TEMPLATES_NAMES, + ]); }); it('adds hidden field correctly', () => { @@ -120,20 +152,24 @@ describe('EPM template', () => { const templateWithHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, hidden: true, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(templateWithHidden.data_stream.hidden).toEqual(true); const templateWithoutHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); @@ -143,6 +179,7 @@ describe('EPM template', () => { const templateWithGlobalAndDataStreamHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, @@ -153,11 +190,13 @@ describe('EPM template', () => { hidden: true, }, }, + isIndexModeTimeSeries: false, }); expect(templateWithGlobalAndDataStreamHidden.data_stream.hidden).toEqual(true); const templateWithDataStreamHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, @@ -167,21 +206,25 @@ describe('EPM template', () => { hidden: true, }, }, + isIndexModeTimeSeries: false, }); expect(templateWithDataStreamHidden.data_stream.hidden).toEqual(true); const templateWithoutDataStreamHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, hidden: true, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(templateWithoutDataStreamHidden.data_stream.hidden).toEqual(true); const templateWithGlobalHiddenTrueAndDataStreamHiddenFalse = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, @@ -192,15 +235,18 @@ describe('EPM template', () => { hidden: false, }, }, + isIndexModeTimeSeries: false, }); expect(templateWithGlobalHiddenTrueAndDataStreamHiddenFalse.data_stream.hidden).toEqual(true); const templateWithoutHidden = getTemplate({ templateIndexPattern, + type: 'logs', packageName: 'nginx', composedOfTemplates: [], templatePriority: 200, mappings: { properties: [] }, + isIndexModeTimeSeries: false, }); expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 36e13ae331f58..26c6926c0bbc4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -27,6 +27,9 @@ import { getRegistryDataStreamAssetBaseName } from '../../../../../common/servic import { FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS, + STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS, } from '../../../../constants'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -76,12 +79,14 @@ export function getTemplate({ registryElasticsearch, mappings, isIndexModeTimeSeries, + type, }: { templateIndexPattern: string; packageName: string; composedOfTemplates: string[]; templatePriority: number; mappings: IndexTemplateMappings; + type: string; hidden?: boolean; registryElasticsearch?: RegistryElasticsearch | undefined; isIndexModeTimeSeries?: boolean; @@ -100,7 +105,10 @@ export function getTemplate({ throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`); } + const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries); + template.composed_of = [ + ...esBaseComponents, ...(template.composed_of || []), FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, ...(appContextService.getConfig()?.agentIdVerificationEnabled @@ -111,6 +119,20 @@ export function getTemplate({ return template; } +const getBaseEsComponents = (type: string, isIndexModeTimeSeries: boolean): string[] => { + if (type === 'metrics') { + if (isIndexModeTimeSeries) { + return [STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS]; + } + + return [STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS]; + } else if (type === 'logs') { + return [STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS]; + } + + return []; +}; + /** * Generate mapping takes the given nested fields array and creates the Elasticsearch * mapping properties out of it. diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index c93f30dddff7e..26211028a3411 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -57,6 +57,7 @@ export default function (providerContext: FtrProviderContext) { // the index template composed_of has the correct component templates in the correct order const indexTemplate = indexTemplateResponse.index_templates[0].index_template; expect(indexTemplate.composed_of).to.eql([ + `logs@settings`, `${templateName}@package`, `${templateName}@custom`, '.fleet_globals-1', @@ -131,13 +132,11 @@ export default function (providerContext: FtrProviderContext) { template: { settings: { index: { - codec: 'best_compression', default_pipeline: 'logs-overrides.test-0.1.0', lifecycle: { name: 'overridden by user', }, mapping: { - ignore_malformed: `true`, total_fields: { limit: '10000', }, @@ -149,7 +148,6 @@ export default function (providerContext: FtrProviderContext) { dynamic: 'false', properties: { '@timestamp': { - ignore_malformed: false, type: 'date', }, data_stream: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/template.ts b/x-pack/test/fleet_api_integration/apis/epm/template.ts index c62afa6dccbf7..8f2d208a27421 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/template.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/template.ts @@ -35,6 +35,7 @@ export default function ({ getService }: FtrProviderContext) { composedOfTemplates: [], templatePriority: 200, mappings: { properties: [] }, + type: 'logs', }); // This test is not an API integration test with Kibana diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 9b2527a5b025d..75040c8400d04 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -217,13 +217,11 @@ export default function (providerContext: FtrProviderContext) { expect(resPackage.statusCode).equal(200); expect(resPackage.body.component_templates[0].component_template.template.settings).eql({ index: { - codec: 'best_compression', default_pipeline: 'logs-all_assets.test_logs-0.2.0', lifecycle: { name: 'reference2', }, mapping: { - ignore_malformed: `true`, total_fields: { limit: '10000', }, diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts b/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts index a1e6fe5d5556f..0d2af1d0e6ac1 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts @@ -221,11 +221,9 @@ export default function (providerContext: FtrProviderContext) { settings: { index: { lifecycle: { name: 'logs' }, - codec: 'best_compression', default_pipeline: 'logs-dataset1-1.0.0', mapping: { total_fields: { limit: '10000' }, - ignore_malformed: 'true', }, }, },